

<!DOCTYPE html>


<html lang="en" data-content_root="" >

  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />

    <title>Brevitas TVMCon 2021 tutorial &#8212; Brevitas Documentation - v0.10.3</title>
  
  
  
  <script data-cfasync="false">
    document.documentElement.dataset.mode = localStorage.getItem("mode") || "";
    document.documentElement.dataset.theme = localStorage.getItem("theme") || "";
  </script>
  
  <!-- Loaded before other Sphinx assets -->
  <link href="../_static/styles/theme.css?digest=3ee479438cf8b5e0d341" rel="stylesheet" />
<link href="../_static/styles/bootstrap.css?digest=3ee479438cf8b5e0d341" rel="stylesheet" />
<link href="../_static/styles/pydata-sphinx-theme.css?digest=3ee479438cf8b5e0d341" rel="stylesheet" />

  
  <link href="../_static/vendor/fontawesome/6.5.2/css/all.min.css?digest=3ee479438cf8b5e0d341" rel="stylesheet" />
  <link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-solid-900.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-brands-400.woff2" />
<link rel="preload" as="font" type="font/woff2" crossorigin href="../_static/vendor/fontawesome/6.5.2/webfonts/fa-regular-400.woff2" />

    <link rel="stylesheet" type="text/css" href="../_static/pygments.css" />
    <link rel="stylesheet" type="text/css" href="../_static/sg_gallery.css" />
    <link rel="stylesheet" type="text/css" href="../_static/nbsphinx-code-cells.css" />
  
  <!-- Pre-loaded scripts that we'll load fully later -->
  <link rel="preload" as="script" href="../_static/scripts/bootstrap.js?digest=3ee479438cf8b5e0d341" />
<link rel="preload" as="script" href="../_static/scripts/pydata-sphinx-theme.js?digest=3ee479438cf8b5e0d341" />
  <script src="../_static/vendor/fontawesome/6.5.2/js/all.min.js?digest=3ee479438cf8b5e0d341"></script>

    <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
    <script src="../_static/jquery.js"></script>
    <script src="../_static/underscore.js"></script>
    <script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
    <script src="../_static/doctools.js"></script>
    <script src="../_static/sphinx_highlight.js"></script>
    <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
    <script>window.MathJax = {"tex": {"inlineMath": [["$", "$"], ["\\(", "\\)"]], "processEscapes": true}, "options": {"ignoreHtmlClass": "tex2jax_ignore|mathjax_ignore|document", "processHtmlClass": "tex2jax_process|mathjax_process|math|output_area"}}</script>
    <script defer="defer" src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
    <script>DOCUMENTATION_OPTIONS.pagename = 'tutorials/tvmcon2021';</script>
    <script>
        DOCUMENTATION_OPTIONS.theme_version = '0.15.3';
        DOCUMENTATION_OPTIONS.theme_switcher_json_url = 'https://xilinx.github.io/brevitas/dev/_static/versions.json';
        DOCUMENTATION_OPTIONS.theme_switcher_version_match = 'v0.10.3';
        DOCUMENTATION_OPTIONS.show_version_warning_banner = false;
        </script>
    <link rel="author" title="About these documents" href="../about.html" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="An overview of QuantTensor and QuantConv2d" href="quant_tensor_quant_conv2d_overview.html" />
    <link rel="prev" title="Tutorials" href="index.html" />
  <meta name="viewport" content="width=device-width, initial-scale=1"/>
  <meta name="docsearch:language" content="en"/>
  </head>
  
  
  <body data-bs-spy="scroll" data-bs-target=".bd-toc-nav" data-offset="180" data-bs-root-margin="0px 0px -60%" data-default-mode="">

  
  
  <div id="pst-skip-link" class="skip-link d-print-none"><a href="#main-content">Skip to main content</a></div>
  
  <div id="pst-scroll-pixel-helper"></div>
  
  <button type="button" class="btn rounded-pill" id="pst-back-to-top">
    <i class="fa-solid fa-arrow-up"></i>Back to top</button>

  
  <input type="checkbox"
          class="sidebar-toggle"
          id="pst-primary-sidebar-checkbox"/>
  <label class="overlay overlay-primary" for="pst-primary-sidebar-checkbox"></label>
  
  <input type="checkbox"
          class="sidebar-toggle"
          id="pst-secondary-sidebar-checkbox"/>
  <label class="overlay overlay-secondary" for="pst-secondary-sidebar-checkbox"></label>
  
  <div class="search-button__wrapper">
    <div class="search-button__overlay"></div>
    <div class="search-button__search-container">
<form class="bd-search d-flex align-items-center"
      action="../search.html"
      method="get">
  <i class="fa-solid fa-magnifying-glass"></i>
  <input type="search"
         class="form-control"
         name="q"
         id="search-input"
         placeholder="Search the docs ..."
         aria-label="Search the docs ..."
         autocomplete="off"
         autocorrect="off"
         autocapitalize="off"
         spellcheck="false"/>
  <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd>K</kbd></span>
</form></div>
  </div>

  <div class="pst-async-banner-revealer d-none">
  <aside id="bd-header-version-warning" class="d-none d-print-none" aria-label="Version warning"></aside>
</div>

  
    <header class="bd-header navbar navbar-expand-lg bd-navbar d-print-none">
<div class="bd-header__inner bd-page-width">
  <button class="sidebar-toggle primary-toggle" aria-label="Site navigation">
    <span class="fa-solid fa-bars"></span>
  </button>
  
  
  <div class="col-lg-3 navbar-header-items__start">
    
      <div class="navbar-item">

  

<a class="navbar-brand logo" href="../index.html">
  
  
  
  
  
    
    
      
    
    
    <img src="../_static/brevitas_logo_black.svg" class="logo__image only-light" alt="Brevitas Documentation - v0.10.3 - Home"/>
    <script>document.write(`<img src="../_static/brevitas_logo_white.svg" class="logo__image only-dark" alt="Brevitas Documentation - v0.10.3 - Home"/>`);</script>
  
  
</a></div>
    
  </div>
  
  <div class="col-lg-9 navbar-header-items">
    
    <div class="me-auto navbar-header-items__center">
      
        <div class="navbar-item">
<nav class="navbar-nav">
  <ul class="bd-navbar-elements navbar-nav">
    
<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../setup.html">
    Setup
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../getting_started.html">
    Getting Started
  </a>
</li>


<li class="nav-item pst-header-nav-item current active">
  <a class="nav-link nav-internal" href="index.html">
    Tutorials
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../settings.html">
    Settings
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../api_reference/index.html">
    API reference
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../architecture.html">
    Architecture
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../faq.html">
    FAQ
  </a>
</li>

            <li class="nav-item dropdown pst-header-nav-item">
                <button class="btn dropdown-toggle nav-item" type="button" data-bs-toggle="dropdown" aria-expanded="false" aria-controls="pst-nav-more-links">
                    More
                </button>
                <ul id="pst-nav-more-links" class="dropdown-menu">
                    
<li class="nav-item ">
  <a class="nav-link dropdown-item nav-internal" href="../about.html">
    About
  </a>
</li>

                </ul>
            </li>
            
  </ul>
</nav></div>
      
    </div>
    
    
    <div class="navbar-header-items__end">
      
        <div class="navbar-item navbar-persistent--container">
          

 <script>
 document.write(`
   <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
    <i class="fa-solid fa-magnifying-glass"></i>
    <span class="search-button__default-text">Search</span>
    <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
   </button>
 `);
 </script>
        </div>
      
      
        <div class="navbar-item">

<script>
document.write(`
  <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
    <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
    <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
    <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
  </button>
`);
</script></div>
      
    </div>
    
  </div>
  
  
    <div class="navbar-persistent--mobile">

 <script>
 document.write(`
   <button class="btn navbar-btn search-button-field search-button__button" title="Search" aria-label="Search" data-bs-placement="bottom" data-bs-toggle="tooltip">
    <i class="fa-solid fa-magnifying-glass"></i>
    <span class="search-button__default-text">Search</span>
    <span class="search-button__kbd-shortcut"><kbd class="kbd-shortcut__modifier">Ctrl</kbd>+<kbd class="kbd-shortcut__modifier">K</kbd></span>
   </button>
 `);
 </script>
    </div>
  

  
    <button class="sidebar-toggle secondary-toggle" aria-label="On this page">
      <span class="fa-solid fa-outdent"></span>
    </button>
  
</div>

    </header>
  

  <div class="bd-container">
    <div class="bd-container__inner bd-page-width">
      
      
      
      <div class="bd-sidebar-primary bd-sidebar">
        

  
  <div class="sidebar-header-items sidebar-primary__section">
    
    
      <div class="sidebar-header-items__center">
        
          
          
            <div class="navbar-item">
<nav class="navbar-nav">
  <ul class="bd-navbar-elements navbar-nav">
    
<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../setup.html">
    Setup
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../getting_started.html">
    Getting Started
  </a>
</li>


<li class="nav-item pst-header-nav-item current active">
  <a class="nav-link nav-internal" href="index.html">
    Tutorials
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../settings.html">
    Settings
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../api_reference/index.html">
    API reference
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../architecture.html">
    Architecture
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../faq.html">
    FAQ
  </a>
</li>


<li class="nav-item pst-header-nav-item">
  <a class="nav-link nav-internal" href="../about.html">
    About
  </a>
</li>

  </ul>
</nav></div>
          
        
      </div>
    
    
    
      <div class="sidebar-header-items__end">
        
          <div class="navbar-item">

<script>
document.write(`
  <button class="btn btn-sm navbar-btn theme-switch-button" title="light/dark" aria-label="light/dark" data-bs-placement="bottom" data-bs-toggle="tooltip">
    <span class="theme-switch nav-link" data-mode="light"><i class="fa-solid fa-sun fa-lg"></i></span>
    <span class="theme-switch nav-link" data-mode="dark"><i class="fa-solid fa-moon fa-lg"></i></span>
    <span class="theme-switch nav-link" data-mode="auto"><i class="fa-solid fa-circle-half-stroke fa-lg"></i></span>
  </button>
`);
</script></div>
        
      </div>
    
  </div>
  
    <div class="sidebar-primary-items__start sidebar-primary__section">
        <div class="sidebar-primary-item">
<nav class="bd-docs-nav bd-links"
     aria-label="Section Navigation">
  <p class="bd-links__title" role="heading" aria-level="1">Section Navigation</p>
  <div class="bd-toc-item navbar-nav"><p aria-level="2" class="caption" role="heading"><span class="caption-text">Tutorials:</span></p>
<ul class="current nav bd-sidenav">
<li class="toctree-l1 current active"><a class="current reference internal" href="#">Brevitas TVMCon 2021 tutorial</a></li>









<li class="toctree-l1"><a class="reference internal" href="quant_tensor_quant_conv2d_overview.html">An overview of QuantTensor and QuantConv2d</a></li>
<li class="toctree-l1"><a class="reference internal" href="quant_activation_overview.html">An Overview of Quantized Activations</a></li>
<li class="toctree-l1"><a class="reference internal" href="anatomy_quantizer.html">Anatomy of a Quantizer</a></li>
<li class="toctree-l1"><a class="reference internal" href="quant_recurrent.html">Quantized RNNs and LSTMs</a></li>
<li class="toctree-l1"><a class="reference internal" href="onnx_export.html">ONNX Export</a></li>
</ul>
</div>
</nav></div>
    </div>
  
  
  <div class="sidebar-primary-items__end sidebar-primary__section">
  </div>
  
  <div id="rtd-footer-container"></div>


      </div>
      
      <main id="main-content" class="bd-main" role="main">
        
        
          <div class="bd-content">
            <div class="bd-article-container">
              
              <div class="bd-header-article d-print-none">
<div class="header-article-items header-article__inner">
  
    <div class="header-article-items__start">
      
        <div class="header-article-item">



<nav aria-label="Breadcrumb" class="d-print-none">
  <ul class="bd-breadcrumbs">
    
    <li class="breadcrumb-item breadcrumb-home">
      <a href="../index.html" class="nav-link" aria-label="Home">
        <i class="fa-solid fa-home"></i>
      </a>
    </li>
    
    <li class="breadcrumb-item"><a href="index.html" class="nav-link">Tutorials</a></li>
    
    <li class="breadcrumb-item active" aria-current="page">Brevitas...</li>
  </ul>
</nav>
</div>
      
    </div>
  
  
</div>
</div>
              
              
              
                
<div id="searchbox"></div>
                <article class="bd-article">
                  
  <section id="Brevitas-TVMCon-2021-tutorial">
<h1>Brevitas TVMCon 2021 tutorial<a class="headerlink" href="#Brevitas-TVMCon-2021-tutorial" title="Permalink to this heading">#</a></h1>
<p>Brevitas requires Python 3.8+ and PyTorch 1.5.0+ and can be installed from PyPI with <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">brevitas</span></code>. This notebook was originally released for Brevitas 0.7.1. It has since been updated to run with Brevitas 0.8 and PyTorch 1.9.0 and is going to be available at <code class="docutils literal notranslate"><span class="pre">https://github.com/Xilinx/brevitas/tree/master/notebooks</span></code>.</p>
</section>
<section id="Fundamentals">
<h1>Fundamentals<a class="headerlink" href="#Fundamentals" title="Permalink to this heading">#</a></h1>
<section id="QuantLinear-layer">
<h2>QuantLinear layer<a class="headerlink" href="#QuantLinear-layer" title="Permalink to this heading">#</a></h2>
<p>We start by looking at <code class="docutils literal notranslate"><span class="pre">brevitas.nn.QuantLinear</span></code>, a quantized alternative to <code class="docutils literal notranslate"><span class="pre">torch.nn.Linear</span></code>, and an instance of a <code class="docutils literal notranslate"><span class="pre">QuantWeightBiasInputOutputLayer</span></code>, meaning that it supports quantization of its weight, bias, input and output. Other instances of <code class="docutils literal notranslate"><span class="pre">QuantWBIOL</span></code> are <code class="docutils literal notranslate"><span class="pre">QuantConv1d</span></code>, <code class="docutils literal notranslate"><span class="pre">QuantConv2d</span></code>, <code class="docutils literal notranslate"><span class="pre">QuantConvTranspose1d</span></code> and <code class="docutils literal notranslate"><span class="pre">QuantConvTranspose2d</span></code>, and they all follow the same principles.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[1]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">inspect</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.nn</span><span class="w"> </span><span class="kn">import</span> <span class="n">QuantLinear</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">IPython.display</span><span class="w"> </span><span class="kn">import</span> <span class="n">Markdown</span><span class="p">,</span> <span class="n">display</span>

<span class="k">def</span><span class="w"> </span><span class="nf">pretty_print_source</span><span class="p">(</span><span class="n">source</span><span class="p">):</span>
    <span class="n">display</span><span class="p">(</span><span class="n">Markdown</span><span class="p">(</span><span class="s1">&#39;```python</span><span class="se">\n</span><span class="s1">&#39;</span> <span class="o">+</span> <span class="n">source</span> <span class="o">+</span> <span class="s1">&#39;</span><span class="se">\n</span><span class="s1">```&#39;</span><span class="p">))</span>

<span class="n">source</span> <span class="o">=</span> <span class="n">inspect</span><span class="o">.</span><span class="n">getsource</span><span class="p">(</span><span class="n">QuantLinear</span><span class="o">.</span><span class="fm">__init__</span><span class="p">)</span>
<span class="n">pretty_print_source</span><span class="p">(</span><span class="n">source</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span><span class="w"> </span><span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">in_features</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">out_features</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">bias</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
        <span class="n">weight_quant</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">WeightQuantType</span><span class="p">]</span> <span class="o">=</span> <span class="n">Int8WeightPerTensorFloat</span><span class="p">,</span>
        <span class="n">bias_quant</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">BiasQuantType</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">input_quant</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ActQuantType</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">output_quant</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">ActQuantType</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">return_quant_tensor</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
    <span class="n">Linear</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">in_features</span><span class="p">,</span> <span class="n">out_features</span><span class="p">,</span> <span class="n">bias</span><span class="p">)</span>
    <span class="n">QuantWBIOL</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">weight_quant</span><span class="o">=</span><span class="n">weight_quant</span><span class="p">,</span>
        <span class="n">bias_quant</span><span class="o">=</span><span class="n">bias_quant</span><span class="p">,</span>
        <span class="n">input_quant</span><span class="o">=</span><span class="n">input_quant</span><span class="p">,</span>
        <span class="n">output_quant</span><span class="o">=</span><span class="n">output_quant</span><span class="p">,</span>
        <span class="n">return_quant_tensor</span><span class="o">=</span><span class="n">return_quant_tensor</span><span class="p">,</span>
        <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
<p>By default <code class="docutils literal notranslate"><span class="pre">weight_quant=Int8WeightPerTensorFloat</span></code>, while <code class="docutils literal notranslate"><span class="pre">bias_quant</span></code>, <code class="docutils literal notranslate"><span class="pre">input_quant</span></code> and <code class="docutils literal notranslate"><span class="pre">output_quant</span></code> are set to <code class="docutils literal notranslate"><span class="pre">None</span></code>, meaning that by default weight quantization is enabled, while everything else is disabled.</p>
</section>
<section id="Weight-quantization">
<h2>Weight quantization<a class="headerlink" href="#Weight-quantization" title="Permalink to this heading">#</a></h2>
<section id="Default-weight-quantization">
<h3>Default weight quantization<a class="headerlink" href="#Default-weight-quantization" title="Permalink to this heading">#</a></h3>
<p><code class="docutils literal notranslate"><span class="pre">weight_quant=Int8WeightPerTensorFloat</span></code> means that by default weights are quantized to <em>8-bit signed integer with a per-tensor floating-point scale factor</em>, while quantization of bias, input, and output are disabled.</p>
<p>We can instantiate a layer and inspect the original weight tensor and the quantized version to see the effect:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[2]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">torch</span>

<span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Original float weight tensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">weight</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quantized weight QuantTensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Original float weight tensor:
 Parameter containing:
tensor([[-0.0053,  0.3793],
        [-0.5820, -0.5204],
        [-0.2723,  0.1896],
        [-0.0140,  0.5607]], requires_grad=True)

Quantized weight QuantTensor:
 QuantTensor(value=tensor([[-0.0046,  0.3803],
        [-0.5820, -0.5224],
        [-0.2704,  0.1879],
        [-0.0137,  0.5591]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0046, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))

</pre></div></div>
</div>
<p>As it can be noticed, by default the scale factor is differentiable and it’s computed based on the maximum absolute value found within the full precision weight tensor.</p>
<p>We can also retrieve the corrisponding integer representation:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[3]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quantized Weight integer tensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="o">.</span><span class="n">int</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Quantized Weight integer tensor:
 tensor([[  -1,   83],
        [-127, -114],
        [ -59,   41],
        [  -3,  122]], dtype=torch.int32)
</pre></div></div>
</div>
</section>
<section id="Mixing-quantized-weights-and-floating-point-inputs">
<h3>Mixing quantized weights and floating-point inputs<a class="headerlink" href="#Mixing-quantized-weights-and-floating-point-inputs" title="Permalink to this heading">#</a></h3>
<p>In any of the layers defined so far we enabled only weight quantization. So by passing in a float input we would get in general an float output, with the linear operation being computed between the unquantized input and the dequantized weights:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[4]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">float_output</span> <span class="o">=</span> <span class="n">quant_linear</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Float input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">float_input</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Float output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">float_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Float input:
 tensor([[ 1.5410, -0.2934],
        [-2.1788,  0.5684],
        [-1.0845, -1.3986]])
Float output:
 tensor([[-0.9036, -0.4586,  0.3096, -0.6472],
        [ 1.2058,  0.6525, -0.3723,  0.8677],
        [ 1.3873,  0.2801, -0.9009,  0.9507]], grad_fn=&lt;MmBackward&gt;)
</pre></div></div>
</div>
<p>In general operations involving quantized tensors are always computed through standard torch operators (here <code class="docutils literal notranslate"><span class="pre">torch.nn.functional.linear</span></code> called internally by the module) on the dequantized representation, the so-called fake-quantization approach.</p>
</section>
<section id="Fixed-point-weight-quantization">
<h3>Fixed-point weight quantization<a class="headerlink" href="#Fixed-point-weight-quantization" title="Permalink to this heading">#</a></h3>
<p>Brevitas exposes various pre-made quantizers. For example, for pure fixed point quantization (i.e. restricting the scale to a power of two) we can use <code class="docutils literal notranslate"><span class="pre">Int8WeightPerTensorFixedPoint</span></code>:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[5]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">import</span><span class="w"> </span><span class="nn">math</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">Int8WeightPerTensorFixedPoint</span>

<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">weight_quant</span><span class="o">=</span><span class="n">Int8WeightPerTensorFixedPoint</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Weight QuantTensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Weight fix point: </span><span class="si">{</span><span class="o">-</span><span class="w"> </span><span class="n">math</span><span class="o">.</span><span class="n">log2</span><span class="p">(</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="o">.</span><span class="n">scale</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Weight QuantTensor:
 QuantTensor(value=tensor([[-0.0078,  0.3828],
        [-0.5781, -0.5234],
        [-0.2734,  0.1875],
        [-0.0156,  0.5625]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0078, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))
Weight fix point: 7.0
</pre></div></div>
</div>
</section>
<section id="Binary-weight-quantization">
<h3>Binary weight quantization<a class="headerlink" href="#Binary-weight-quantization" title="Permalink to this heading">#</a></h3>
<p>For binary quantization with a constant scale factor (defaults 0.1) we can set the <code class="docutils literal notranslate"><span class="pre">SignedBinaryWeightPerTensorConst</span></code> quantizer:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[6]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">SignedBinaryWeightPerTensorConst</span>

<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">weight_quant</span><span class="o">=</span><span class="n">SignedBinaryWeightPerTensorConst</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Weight QuantTensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Weight QuantTensor:
 QuantTensor(value=tensor([[-0.1000,  0.1000],
        [-0.1000, -0.1000],
        [-0.1000,  0.1000],
        [-0.1000,  0.1000]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.1000), zero_point=tensor(0.), bit_width=tensor(1.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
</section>
<section id="Sharing-a-weight-quantizer">
<h3>Sharing a weight quantizer<a class="headerlink" href="#Sharing-a-weight-quantizer" title="Permalink to this heading">#</a></h3>
<p>Brevitas also allows to share <strong>instances</strong> (not definitions) of weight quantizers among layers. This is different from applying the same quantization strategy to multiple layers, and it forces them to have the same scale, zero-point, and bit-width. It can be handy in a variety of situations as we will see later, and it works across different quantization strategies.</p>
<p>It’s designed to operate in an eager-mode friendly way, meaning that the number of layers sharing it doesn’t have to be known in advance. The quantizer is re-initialized appropriately any time it’s shared to a new layer.</p>
<p>For example, we mentioned that the default weight quantizer <code class="docutils literal notranslate"><span class="pre">Int8WeightPerTensorFloat</span></code> computes the scale factor based on the maximum value found within the floating-point weight tensor to quantize. Sharing it among multiple layers means that the quantizer now looks at all the weight tensors that are being quantized to determine the overall maximum value and generate a single scale factor:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[7]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="c1"># Define a QuantLinear layer 1</span>
<span class="n">quant_linear1</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="c1"># Keep a pointer to the scale factor of QuantLinear layer 1 weights before sharing</span>
<span class="n">quant_linear1_scale_before_sharing</span> <span class="o">=</span> <span class="n">quant_linear1</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="o">.</span><span class="n">scale</span>

<span class="c1"># Define a QuantLinear layer 2 where the weight quantizer is taken from layer 1</span>
<span class="n">quant_linear2</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">weight_quant</span><span class="o">=</span><span class="n">quant_linear1</span><span class="o">.</span><span class="n">weight_quant</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;QuantLinear 1 scale before sharing with QuantLinear 2: </span><span class="si">{</span><span class="n">quant_linear1_scale_before_sharing</span><span class="si">:</span><span class="s2">.4f</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;QuantLinear 2 scale: </span><span class="si">{</span><span class="n">quant_linear2</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="o">.</span><span class="n">scale</span><span class="si">:</span><span class="s2">.4f</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;QuantLinear 1 scale after sharing with QuantLinear 2: </span><span class="si">{</span><span class="n">quant_linear1</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="o">.</span><span class="n">scale</span><span class="si">:</span><span class="s2">.4f</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<br/></pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
QuantLinear 1 scale before sharing with QuantLinear 2: 0.0046
QuantLinear 2 scale: 0.0053
QuantLinear 1 scale after sharing with QuantLinear 2: 0.0053
</pre></div></div>
</div>
</section>
</section>
<section id="Inputs/Outputs/Activations-quantization:">
<h2>Inputs/Outputs/Activations quantization:<a class="headerlink" href="#Inputs/Outputs/Activations-quantization:" title="Permalink to this heading">#</a></h2>
<p>Let’s say now we want to quantize the input to <code class="docutils literal notranslate"><span class="pre">QuantLinear</span></code> to generate a quantized output. We can set an appropriate input quantizer like <code class="docutils literal notranslate"><span class="pre">Int8ActPerTensorFloat</span></code>:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[8]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">Int8ActPerTensorFloat</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">input_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloat</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="n">quant_output</span> <span class="o">=</span> <span class="n">quant_linear</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Float input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">float_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Float input:
 tensor([[ 1.5410, -0.2934],
        [-2.1788,  0.5684],
        [-1.0845, -1.3986]])

Quant output:
 tensor([[-0.9109, -0.4609,  0.3135, -0.6523],
        [ 1.2089,  0.6524, -0.3752,  0.8697],
        [ 1.3893,  0.2816, -0.9011,  0.9521]], grad_fn=&lt;MmBackward&gt;)
</pre></div></div>
</div>
<p>Note how by default the output tensor is returned as a standard torch tensor in dequantized format. This is designed to minimize friction with standard torch operators in the default case. To return an output <code class="docutils literal notranslate"><span class="pre">QuantTensor</span></code> we have to set <code class="docutils literal notranslate"><span class="pre">return_quant_tensor=True</span></code>:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[9]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">Int8ActPerTensorFloat</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">input_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloat</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_output</span> <span class="o">=</span> <span class="n">quant_linear</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Quant output:
 QuantTensor(value=tensor([[-0.9109, -0.4609,  0.3135, -0.6523],
        [ 1.2089,  0.6524, -0.3752,  0.8697],
        [ 1.3893,  0.2816, -0.9011,  0.9521]], grad_fn=&lt;MmBackward&gt;), scale=tensor([[9.0542e-05]], grad_fn=&lt;MulBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
<section id="QuantIdentity-layer">
<h3>QuantIdentity layer<a class="headerlink" href="#QuantIdentity-layer" title="Permalink to this heading">#</a></h3>
<p>Alternatively we could have instantiated a <code class="docutils literal notranslate"><span class="pre">QuantIdentity</span></code> layer. By default <code class="docutils literal notranslate"><span class="pre">QuantIdentity</span></code> adopts the <code class="docutils literal notranslate"><span class="pre">Int8ActPerTensorFloat</span></code> quantizer:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[10]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.nn</span><span class="w"> </span><span class="kn">import</span> <span class="n">QuantIdentity</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_identity</span> <span class="o">=</span> <span class="n">QuantIdentity</span><span class="p">(</span><span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_input</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>
<span class="n">quant_output</span> <span class="o">=</span> <span class="n">quant_linear</span><span class="p">(</span><span class="n">quant_input</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Float input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">float_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Float input:
 tensor([[ 1.5410, -0.2934],
        [-2.1788,  0.5684],
        [-1.0845, -1.3986]])

Quant input:
 QuantTensor(value=tensor([[ 1.5490, -0.2894],
        [-2.1788,  0.5617],
        [-1.0894, -1.3958]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0170, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))

Quant output:
 QuantTensor(value=tensor([[-0.9109, -0.4609,  0.3135, -0.6523],
        [ 1.2089,  0.6524, -0.3752,  0.8697],
        [ 1.3893,  0.2816, -0.9011,  0.9521]], grad_fn=&lt;MmBackward&gt;), scale=tensor([[9.0542e-05]], grad_fn=&lt;MulBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
<p>Note that Having the input/output quantizer as part of a layer like <code class="docutils literal notranslate"><span class="pre">QuantLinear</span></code> or as a standalone <code class="docutils literal notranslate"><span class="pre">QuantIdentity</span></code> can make a difference at export time with targets like e.g. the standard ONNX opset, where <code class="docutils literal notranslate"><span class="pre">QLinearMatMul</span></code> or <code class="docutils literal notranslate"><span class="pre">QLinearConv</span></code> assume the output quantizer is part of the layer.</p>
</section>
<section id="QuantReLU-layer">
<h3>QuantReLU layer<a class="headerlink" href="#QuantReLU-layer" title="Permalink to this heading">#</a></h3>
<p>For QuantReLU the default quantizer is <code class="docutils literal notranslate"><span class="pre">Uint8ActPerTensorFloat</span></code>. Quantized activations layers like <code class="docutils literal notranslate"><span class="pre">QuantReLU</span></code> by defaults quantize the output, meaning that a relu is computed first, and then the output is quantized:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[11]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.nn</span><span class="w"> </span><span class="kn">import</span> <span class="n">QuantReLU</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_relu</span> <span class="o">=</span> <span class="n">QuantReLU</span><span class="p">(</span><span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_output</span> <span class="o">=</span> <span class="n">quant_relu</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Float input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">float_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Float input:
 tensor([[ 1.5410, -0.2934],
        [-2.1788,  0.5684],
        [-1.0845, -1.3986]])

Quant output:
 QuantTensor(value=tensor([[1.5410, 0.0000],
        [0.0000, 0.5681],
        [0.0000, 0.0000]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0060, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(False), training_t=tensor(True))
</pre></div></div>
</div>
</section>
<section id="Requantizing-a-tensor">
<h3>Requantizing a tensor<a class="headerlink" href="#Requantizing-a-tensor" title="Permalink to this heading">#</a></h3>
<p>As you might expect, we can pass a <code class="docutils literal notranslate"><span class="pre">QuantTensor</span></code> to a quantized activation, and the tensor will be requantized:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[12]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_identity</span> <span class="o">=</span> <span class="n">QuantIdentity</span><span class="p">(</span><span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">quant_relu</span> <span class="o">=</span> <span class="n">QuantReLU</span><span class="p">(</span><span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">signed_quant_output</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>
<span class="n">unsigned_quant_output</span> <span class="o">=</span> <span class="n">quant_relu</span><span class="p">(</span><span class="n">signed_quant_output</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Float input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">float_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant output after QuantIdentity:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">signed_quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant output after QuantReLU:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">unsigned_quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Float input:
 tensor([[ 1.5410, -0.2934],
        [-2.1788,  0.5684],
        [-1.0845, -1.3986]])

Quant output after QuantIdentity:
 QuantTensor(value=tensor([[ 1.5490, -0.2894],
        [-2.1788,  0.5617],
        [-1.0894, -1.3958]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0170, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))
Quant output after QuantReLU:
 QuantTensor(value=tensor([[1.5490, 0.0000],
        [0.0000, 0.5588],
        [0.0000, 0.0000]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0061, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(False), training_t=tensor(True))
</pre></div></div>
</div>
</section>
<section id="How-is-the-activation-scale-determined-by-default?">
<h3>How is the activation scale determined by default?<a class="headerlink" href="#How-is-the-activation-scale-determined-by-default?" title="Permalink to this heading">#</a></h3>
<p>To minimize user interaction, default quantizers like <code class="docutils literal notranslate"><span class="pre">Uint8ActPerTensorFloat</span></code> and <code class="docutils literal notranslate"><span class="pre">Int8ActPerTensorFloat</span></code> initializes activations’ scale by collecting statistics for a number of training steps (by default the 99.999 percentile of the absolute value for 300 steps).</p>
<p>This can be seen as an initial calibration step, although by default it happens with quantization already enabled, while calibration typically collects floating-point statistics first, and the enables quantization. These statistics are accumulated in an exponential moving average that at end of the collection phase is used to internally initialize a learned <code class="docutils literal notranslate"><span class="pre">nn.Parameter</span></code>. It’s also possible to delay enabling quantization and perform proper calibration, but we will cover that later.</p>
<p>During the collection phase the quantizer behaves differently between <code class="docutils literal notranslate"><span class="pre">train()</span></code> and <code class="docutils literal notranslate"><span class="pre">eval()</span></code> mode, similarly to what happens for batch normalization. In <code class="docutils literal notranslate"><span class="pre">train()</span></code> mode, the statistics for that particular batch are returned. In <code class="docutils literal notranslate"><span class="pre">eval()</span></code> mode, the exponential moving average is returned. After the collection phase is over the learned parameter is returned in both execution modes.</p>
</section>
</section>
<section id="Bias-Quantization">
<h2>Bias Quantization<a class="headerlink" href="#Bias-Quantization" title="Permalink to this heading">#</a></h2>
<p>In many inference toolchains, bias is assumed to be quantized with scale factor equal to <code class="docutils literal notranslate"><span class="pre">input_scale</span> <span class="pre">*</span> <span class="pre">weight_scale</span></code>, which means that we need a quantized input somehow. A predefined bias quantizer that reflects that assumption is <code class="docutils literal notranslate"><span class="pre">brevitas.quant.scaled_int.Int16Bias</span></code>. If we simply tried to set it to a <code class="docutils literal notranslate"><span class="pre">QuantLinear</span></code> without any sort of input quantization, we would get an error:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[13]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant.scaled_int</span><span class="w"> </span><span class="kn">import</span> <span class="n">Int16Bias</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">bias_quant</span><span class="o">=</span><span class="n">Int16Bias</span><span class="p">,</span> <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_output</span> <span class="o">=</span> <span class="n">quant_linear</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
<span class="ansi-red-intense-fg ansi-bold">---------------------------------------------------------------------------</span>
<span class="ansi-red-intense-fg ansi-bold">RuntimeError</span>                              Traceback (most recent call last)
<span class="ansi-green-intense-fg ansi-bold">C:\Users\ALESSA~1\AppData\Local\Temp/ipykernel_18920/2660651517.py</span> in <span class="ansi-cyan-fg">&lt;module&gt;</span>
<span class="ansi-green-fg">      6</span> quant_linear <span class="ansi-yellow-intense-fg ansi-bold">=</span> QuantLinear<span class="ansi-yellow-intense-fg ansi-bold">(</span><span class="ansi-cyan-intense-fg ansi-bold">2</span><span class="ansi-yellow-intense-fg ansi-bold">,</span> <span class="ansi-cyan-intense-fg ansi-bold">4</span><span class="ansi-yellow-intense-fg ansi-bold">,</span> bias<span class="ansi-yellow-intense-fg ansi-bold">=</span><span class="ansi-green-intense-fg ansi-bold">True</span><span class="ansi-yellow-intense-fg ansi-bold">,</span> bias_quant<span class="ansi-yellow-intense-fg ansi-bold">=</span>Int16Bias<span class="ansi-yellow-intense-fg ansi-bold">,</span> return_quant_tensor<span class="ansi-yellow-intense-fg ansi-bold">=</span><span class="ansi-green-intense-fg ansi-bold">True</span><span class="ansi-yellow-intense-fg ansi-bold">)</span>
<span class="ansi-green-fg">      7</span>
<span class="ansi-green-intense-fg ansi-bold">----&gt; 8</span><span class="ansi-yellow-intense-fg ansi-bold"> </span>quant_output <span class="ansi-yellow-intense-fg ansi-bold">=</span> quant_linear<span class="ansi-yellow-intense-fg ansi-bold">(</span>float_input<span class="ansi-yellow-intense-fg ansi-bold">)</span>

<span class="ansi-green-intense-fg ansi-bold">~\miniconda3\envs\pt190\lib\site-packages\torch\nn\modules\module.py</span> in <span class="ansi-cyan-fg">_call_impl</span><span class="ansi-blue-intense-fg ansi-bold">(self, *input, **kwargs)</span>
<span class="ansi-green-fg">   1049</span>         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
<span class="ansi-green-fg">   1050</span>                 or _global_forward_hooks or _global_forward_pre_hooks):
<span class="ansi-green-intense-fg ansi-bold">-&gt; 1051</span><span class="ansi-yellow-intense-fg ansi-bold">             </span><span class="ansi-green-intense-fg ansi-bold">return</span> forward_call<span class="ansi-yellow-intense-fg ansi-bold">(</span><span class="ansi-yellow-intense-fg ansi-bold">*</span>input<span class="ansi-yellow-intense-fg ansi-bold">,</span> <span class="ansi-yellow-intense-fg ansi-bold">**</span>kwargs<span class="ansi-yellow-intense-fg ansi-bold">)</span>
<span class="ansi-green-fg">   1052</span>         <span class="ansi-red-intense-fg ansi-bold"># Do not call functions when jit is used</span>
<span class="ansi-green-fg">   1053</span>         full_backward_hooks<span class="ansi-yellow-intense-fg ansi-bold">,</span> non_full_backward_hooks <span class="ansi-yellow-intense-fg ansi-bold">=</span> <span class="ansi-yellow-intense-fg ansi-bold">[</span><span class="ansi-yellow-intense-fg ansi-bold">]</span><span class="ansi-yellow-intense-fg ansi-bold">,</span> <span class="ansi-yellow-intense-fg ansi-bold">[</span><span class="ansi-yellow-intense-fg ansi-bold">]</span>

<span class="ansi-green-intense-fg ansi-bold">c:\users\alessandro\documenti\brevitas_tvmcon\src\brevitas\nn\quant_linear.py</span> in <span class="ansi-cyan-fg">forward</span><span class="ansi-blue-intense-fg ansi-bold">(self, input)</span>
<span class="ansi-green-fg">     96</span>
<span class="ansi-green-fg">     97</span>     <span class="ansi-green-intense-fg ansi-bold">def</span> forward<span class="ansi-yellow-intense-fg ansi-bold">(</span>self<span class="ansi-yellow-intense-fg ansi-bold">,</span> input<span class="ansi-yellow-intense-fg ansi-bold">:</span> Union<span class="ansi-yellow-intense-fg ansi-bold">[</span>Tensor<span class="ansi-yellow-intense-fg ansi-bold">,</span> QuantTensor<span class="ansi-yellow-intense-fg ansi-bold">]</span><span class="ansi-yellow-intense-fg ansi-bold">)</span> <span class="ansi-yellow-intense-fg ansi-bold">-&gt;</span> Union<span class="ansi-yellow-intense-fg ansi-bold">[</span>Tensor<span class="ansi-yellow-intense-fg ansi-bold">,</span> QuantTensor<span class="ansi-yellow-intense-fg ansi-bold">]</span><span class="ansi-yellow-intense-fg ansi-bold">:</span>
<span class="ansi-green-intense-fg ansi-bold">---&gt; 98</span><span class="ansi-yellow-intense-fg ansi-bold">         </span><span class="ansi-green-intense-fg ansi-bold">return</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>forward_impl<span class="ansi-yellow-intense-fg ansi-bold">(</span>input<span class="ansi-yellow-intense-fg ansi-bold">)</span>
<span class="ansi-green-fg">     99</span>
<span class="ansi-green-fg">    100</span>     <span class="ansi-green-intense-fg ansi-bold">def</span> inner_forward_impl<span class="ansi-yellow-intense-fg ansi-bold">(</span>self<span class="ansi-yellow-intense-fg ansi-bold">,</span> x<span class="ansi-yellow-intense-fg ansi-bold">:</span> Tensor<span class="ansi-yellow-intense-fg ansi-bold">,</span> quant_weight<span class="ansi-yellow-intense-fg ansi-bold">:</span> Tensor<span class="ansi-yellow-intense-fg ansi-bold">,</span> quant_bias<span class="ansi-yellow-intense-fg ansi-bold">:</span> Optional<span class="ansi-yellow-intense-fg ansi-bold">[</span>Tensor<span class="ansi-yellow-intense-fg ansi-bold">]</span><span class="ansi-yellow-intense-fg ansi-bold">)</span><span class="ansi-yellow-intense-fg ansi-bold">:</span>

<span class="ansi-green-intense-fg ansi-bold">c:\users\alessandro\documenti\brevitas_tvmcon\src\brevitas\nn\quant_layer.py</span> in <span class="ansi-cyan-fg">forward_impl</span><span class="ansi-blue-intense-fg ansi-bold">(self, inp)</span>
<span class="ansi-green-fg">    355</span>
<span class="ansi-green-fg">    356</span>         <span class="ansi-green-intense-fg ansi-bold">if</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>bias <span class="ansi-green-intense-fg ansi-bold">is</span> <span class="ansi-green-intense-fg ansi-bold">not</span> <span class="ansi-green-intense-fg ansi-bold">None</span><span class="ansi-yellow-intense-fg ansi-bold">:</span>
<span class="ansi-green-intense-fg ansi-bold">--&gt; 357</span><span class="ansi-yellow-intense-fg ansi-bold">             </span>quant_bias <span class="ansi-yellow-intense-fg ansi-bold">=</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>bias_quant<span class="ansi-yellow-intense-fg ansi-bold">(</span>self<span class="ansi-yellow-intense-fg ansi-bold">.</span>bias<span class="ansi-yellow-intense-fg ansi-bold">,</span> output_scale<span class="ansi-yellow-intense-fg ansi-bold">,</span> output_bit_width<span class="ansi-yellow-intense-fg ansi-bold">)</span>
<span class="ansi-green-fg">    358</span>             <span class="ansi-green-intense-fg ansi-bold">if</span> <span class="ansi-green-intense-fg ansi-bold">not</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>training <span class="ansi-green-intense-fg ansi-bold">and</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>cache_inference_quant_bias<span class="ansi-yellow-intense-fg ansi-bold">:</span>
<span class="ansi-green-fg">    359</span>                 self<span class="ansi-yellow-intense-fg ansi-bold">.</span>_cached_bias <span class="ansi-yellow-intense-fg ansi-bold">=</span> _CachedIO<span class="ansi-yellow-intense-fg ansi-bold">(</span>quant_bias<span class="ansi-yellow-intense-fg ansi-bold">.</span>detach<span class="ansi-yellow-intense-fg ansi-bold">(</span><span class="ansi-yellow-intense-fg ansi-bold">)</span><span class="ansi-yellow-intense-fg ansi-bold">,</span> metadata_only<span class="ansi-yellow-intense-fg ansi-bold">=</span><span class="ansi-green-intense-fg ansi-bold">False</span><span class="ansi-yellow-intense-fg ansi-bold">)</span>

<span class="ansi-green-intense-fg ansi-bold">~\miniconda3\envs\pt190\lib\site-packages\torch\nn\modules\module.py</span> in <span class="ansi-cyan-fg">_call_impl</span><span class="ansi-blue-intense-fg ansi-bold">(self, *input, **kwargs)</span>
<span class="ansi-green-fg">   1049</span>         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks
<span class="ansi-green-fg">   1050</span>                 or _global_forward_hooks or _global_forward_pre_hooks):
<span class="ansi-green-intense-fg ansi-bold">-&gt; 1051</span><span class="ansi-yellow-intense-fg ansi-bold">             </span><span class="ansi-green-intense-fg ansi-bold">return</span> forward_call<span class="ansi-yellow-intense-fg ansi-bold">(</span><span class="ansi-yellow-intense-fg ansi-bold">*</span>input<span class="ansi-yellow-intense-fg ansi-bold">,</span> <span class="ansi-yellow-intense-fg ansi-bold">**</span>kwargs<span class="ansi-yellow-intense-fg ansi-bold">)</span>
<span class="ansi-green-fg">   1052</span>         <span class="ansi-red-intense-fg ansi-bold"># Do not call functions when jit is used</span>
<span class="ansi-green-fg">   1053</span>         full_backward_hooks<span class="ansi-yellow-intense-fg ansi-bold">,</span> non_full_backward_hooks <span class="ansi-yellow-intense-fg ansi-bold">=</span> <span class="ansi-yellow-intense-fg ansi-bold">[</span><span class="ansi-yellow-intense-fg ansi-bold">]</span><span class="ansi-yellow-intense-fg ansi-bold">,</span> <span class="ansi-yellow-intense-fg ansi-bold">[</span><span class="ansi-yellow-intense-fg ansi-bold">]</span>

<span class="ansi-green-intense-fg ansi-bold">c:\users\alessandro\documenti\brevitas_tvmcon\src\brevitas\proxy\parameter_quant.py</span> in <span class="ansi-cyan-fg">forward</span><span class="ansi-blue-intense-fg ansi-bold">(self, x, input_scale, input_bit_width)</span>
<span class="ansi-green-fg">    194</span>             impl <span class="ansi-yellow-intense-fg ansi-bold">=</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>export_handler <span class="ansi-green-intense-fg ansi-bold">if</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>export_mode <span class="ansi-green-intense-fg ansi-bold">else</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>tensor_quant
<span class="ansi-green-fg">    195</span>             <span class="ansi-green-intense-fg ansi-bold">if</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>requires_input_scale <span class="ansi-green-intense-fg ansi-bold">and</span> input_scale <span class="ansi-green-intense-fg ansi-bold">is</span> <span class="ansi-green-intense-fg ansi-bold">None</span><span class="ansi-yellow-intense-fg ansi-bold">:</span>
<span class="ansi-green-intense-fg ansi-bold">--&gt; 196</span><span class="ansi-yellow-intense-fg ansi-bold">                 </span><span class="ansi-green-intense-fg ansi-bold">raise</span> RuntimeError<span class="ansi-yellow-intense-fg ansi-bold">(</span><span class="ansi-blue-intense-fg ansi-bold">&#34;Input scale required&#34;</span><span class="ansi-yellow-intense-fg ansi-bold">)</span>
<span class="ansi-green-fg">    197</span>             <span class="ansi-green-intense-fg ansi-bold">if</span> self<span class="ansi-yellow-intense-fg ansi-bold">.</span>requires_input_bit_width <span class="ansi-green-intense-fg ansi-bold">and</span> input_bit_width <span class="ansi-green-intense-fg ansi-bold">is</span> <span class="ansi-green-intense-fg ansi-bold">None</span><span class="ansi-yellow-intense-fg ansi-bold">:</span>
<span class="ansi-green-fg">    198</span>                 <span class="ansi-green-intense-fg ansi-bold">raise</span> RuntimeError<span class="ansi-yellow-intense-fg ansi-bold">(</span><span class="ansi-blue-intense-fg ansi-bold">&#34;Input bit-width required&#34;</span><span class="ansi-yellow-intense-fg ansi-bold">)</span>

<span class="ansi-red-intense-fg ansi-bold">RuntimeError</span>: Input scale required
</pre></div></div>
</div>
<p>We can solve the issue by passing in a <code class="docutils literal notranslate"><span class="pre">QuantTensor</span></code> coming from a different layer as input, or by setting an input quantizer:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[14]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">input_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloat</span><span class="p">,</span> <span class="n">bias_quant</span><span class="o">=</span><span class="n">Int16Bias</span><span class="p">,</span> <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_linear</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[14]:
</pre></div>
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
QuantTensor(value=tensor([[-0.6541,  0.1263,  0.1680, -0.1231],
        [ 1.4658,  1.2395, -0.5207,  1.3989],
        [ 1.6461,  0.8687, -1.0466,  1.4813]], grad_fn=&lt;AddmmBackward&gt;), scale=tensor([[9.0542e-05]], grad_fn=&lt;MulBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(18.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
</section>
</section>
<section id="Operations-on-QuantTensor">
<h1>Operations on QuantTensor<a class="headerlink" href="#Operations-on-QuantTensor" title="Permalink to this heading">#</a></h1>
<section id="Element-wise-Arithmetic">
<h2>Element-wise Arithmetic<a class="headerlink" href="#Element-wise-Arithmetic" title="Permalink to this heading">#</a></h2>
<p>Some basic arithmetic operations on QuantTensor are supported. Depending on the operation, some constraints can apply.</p>
<section id="Element-wise-adds">
<h3>Element-wise adds<a class="headerlink" href="#Element-wise-adds" title="Permalink to this heading">#</a></h3>
<p>For element-wise addition, in line with traditional <em>fixed-point arithmetic</em>, the requirement is that scale of the operands is the same.</p>
<p>In order to make it work with the way quantized activation operates by default during the statistics collection phase, this restriction is enforced in <code class="docutils literal notranslate"><span class="pre">eval()</span></code> mode but not in <code class="docutils literal notranslate"><span class="pre">train()</span></code> mode (during which the operands’ scales are averaged).</p>
<p>Calling the same QuantIdentity on tensors that need to be alligned is an easy way to do so:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[15]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_inp1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">float_inp2</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_identity</span> <span class="o">=</span> <span class="n">QuantIdentity</span><span class="p">(</span><span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="c1">#Training mode, statistics are being collected, scaling factors are different but it doesn&#39;t raise an error</span>
<span class="n">train_quant_inp1</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp1</span><span class="p">)</span>
<span class="n">train_quant_inp2</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp2</span><span class="p">)</span>
<span class="n">train_mode_add</span> <span class="o">=</span> <span class="n">train_quant_inp1</span> <span class="o">+</span> <span class="n">train_quant_inp2</span>

<span class="c1">#Inference mode, the EMA buffer is being used, scaling factors are the same</span>
<span class="n">quant_identity</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
<span class="n">eval_quant_inp1</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp1</span><span class="p">)</span>
<span class="n">eval_quant_inp2</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp2</span><span class="p">)</span>
<span class="n">eval_mode_add</span> <span class="o">=</span> <span class="n">eval_quant_inp1</span> <span class="o">+</span> <span class="n">eval_quant_inp2</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Eval mode add quant inputs:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">eval_quant_inp1</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">eval_quant_inp2</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Eval mode add quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">eval_mode_add</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Eval mode add quant inputs:
 QuantTensor(value=tensor([[ 1.5335, -0.2875],
        [-2.0447,  0.5751],
        [-1.0863, -1.4057]]), scale=tensor(0.0160), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(False))
 QuantTensor(value=tensor([[ 0.3994,  0.8307],
        [-0.7188, -0.3994],
        [-0.5910,  0.1757]]), scale=tensor(0.0160), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(False))

Eval mode add quant output:
 QuantTensor(value=tensor([[ 1.9329,  0.5431],
        [-2.7636,  0.1757],
        [-1.6773, -1.2300]]), scale=tensor(0.0160), zero_point=tensor(0.), bit_width=tensor(9.), signed_t=tensor(True), training_t=tensor(False))
</pre></div></div>
</div>
</section>
</section>
<section id="Calling-torch-functions">
<h2>Calling torch functions<a class="headerlink" href="#Calling-torch-functions" title="Permalink to this heading">#</a></h2>
<p>Through the <code class="docutils literal notranslate"><span class="pre">__torch_function__</span></code> interface (supported on PyTorch &gt;= 1.5.0), standard torch functions can be called on a <code class="docutils literal notranslate"><span class="pre">QuantTensor</span></code>. Currently, for supported operations that are invariant to affine quantization a <code class="docutils literal notranslate"><span class="pre">QuantTensor</span></code> is returned, otherwise the output decays to a floating-point <code class="docutils literal notranslate"><span class="pre">torch.Tensor</span></code>.</p>
<p>In this context for a function <code class="docutils literal notranslate"><span class="pre">func</span></code> to be invariant to quantization means that the output of the function applied to the dequantized value should still be a dequantized value with the same scale, zero-point, and bit-width, i.e. given an input dequantized value where <code class="docutils literal notranslate"><span class="pre">input_dequant_value</span> <span class="pre">/</span> <span class="pre">scale</span> <span class="pre">+</span> <span class="pre">zero_point</span> <span class="pre">==</span> <span class="pre">input_integer_value</span></code> that can be represented within <code class="docutils literal notranslate"><span class="pre">bit_width</span></code> bits, <code class="docutils literal notranslate"><span class="pre">func(input_dequant_value)</span> <span class="pre">/</span> <span class="pre">scale</span> <span class="pre">+</span> <span class="pre">zero_point</span> <span class="pre">==</span> <span class="pre">output_integer_value</span></code> that can also be represented within
<code class="docutils literal notranslate"><span class="pre">bit_width</span></code> bits.</p>
<section id="max_pool-on-QuantTensor">
<h3>max_pool on QuantTensor<a class="headerlink" href="#max_pool-on-QuantTensor" title="Permalink to this heading">#</a></h3>
<p>For example, <code class="docutils literal notranslate"><span class="pre">torch.nn.functional.max_pool1d</span></code> is invariant to quantization:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[16]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="n">quant_identity</span> <span class="o">=</span> <span class="n">QuantIdentity</span><span class="p">(</span><span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_input</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp</span><span class="p">)</span>
<span class="n">quant_output</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">functional</span><span class="o">.</span><span class="n">max_pool1d</span><span class="p">(</span><span class="n">quant_input</span><span class="p">,</span> <span class="n">kernel_size</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">stride</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Quant input:
 QuantTensor(value=tensor([[[-1.1218, -1.1580, -0.2533, -0.4343],
         [ 0.8504,  0.6876, -0.3076, -2.1170]],

        [[ 0.4704, -0.1628,  1.4475,  0.2714],
         [ 0.1628,  0.8685, -0.1448, -0.1086]],

        [[ 0.9228,  1.2666,  2.0084,  0.0543],
         [ 0.6152, -0.4162, -0.8323, -2.3160]]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0181, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))

Quant output:
 QuantTensor(value=tensor([[[-1.1218, -0.2533],
         [ 0.8504, -0.3076]],

        [[ 0.4704,  1.4475],
         [ 0.8685, -0.1086]],

        [[ 1.2666,  2.0084],
         [ 0.6152, -0.8323]]], grad_fn=&lt;SqueezeBackward1&gt;), scale=tensor(0.0181, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area stderr docutils container">
<div class="highlight"><pre>
C:\Users\Alessandro\miniconda3\envs\pt190\lib\site-packages\torch\nn\functional.py:652: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at  ..\c10/core/TensorImpl.h:1156.)
  return torch.max_pool1d(input, kernel_size, stride, padding, dilation, ceil_mode)
</pre></div></div>
</div>
</section>
<section id="tanh-on-QuantTensor">
<h3>tanh on QuantTensor<a class="headerlink" href="#tanh-on-QuantTensor" title="Permalink to this heading">#</a></h3>
<p>While for example <code class="docutils literal notranslate"><span class="pre">torch.tanh</span></code> is not:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[17]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">)</span>
<span class="n">quant_identity</span> <span class="o">=</span> <span class="n">QuantIdentity</span><span class="p">(</span><span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_input</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp</span><span class="p">)</span>
<span class="n">quant_output</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tanh</span><span class="p">(</span><span class="n">quant_input</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Quant input:
 QuantTensor(value=tensor([[[-1.1218, -1.1580, -0.2533, -0.4343],
         [ 0.8504,  0.6876, -0.3076, -2.1170]],

        [[ 0.4704, -0.1628,  1.4475,  0.2714],
         [ 0.1628,  0.8685, -0.1448, -0.1086]],

        [[ 0.9228,  1.2666,  2.0084,  0.0543],
         [ 0.6152, -0.4162, -0.8323, -2.3160]]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0181, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(True))

Quant output:
 tensor([[[-0.8082, -0.8204, -0.2480, -0.4089],
         [ 0.6913,  0.5964, -0.2983, -0.9714]],

        [[ 0.4386, -0.1614,  0.8952,  0.2649],
         [ 0.1614,  0.7006, -0.1438, -0.1081]],

        [[ 0.7272,  0.8529,  0.9646,  0.0542],
         [ 0.5478, -0.3937, -0.6817, -0.9807]]], grad_fn=&lt;TanhBackward&gt;)
</pre></div></div>
</div>
</section>
<section id="QuantTensor-concatenation">
<h3>QuantTensor concatenation<a class="headerlink" href="#QuantTensor-concatenation" title="Permalink to this heading">#</a></h3>
<p>Concatenation, i.e. <code class="docutils literal notranslate"><span class="pre">torch.cat</span></code>, can also be applied to quantized tensors whenever they all have the same sign, scale, zero-point and bit-width. Simiarly to element-wise adds, scale and zero-point are allowed to be different in training mode, but have to be the same in inference mode. Example:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[18]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_inp1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">float_inp2</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_identity</span> <span class="o">=</span> <span class="n">QuantIdentity</span><span class="p">(</span><span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="c1">#Training mode, statistics are being collected, scaling factors are different but it doesn&#39;t raise an error</span>
<span class="n">train_mode_cat</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp1</span><span class="p">),</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp2</span><span class="p">)],</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

<span class="c1">#Inference mode, the EMA buffer is being used, scaling factors are the same</span>
<span class="n">quant_identity</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
<span class="n">eval_quant_inp1</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp1</span><span class="p">)</span>
<span class="n">eval_quant_inp2</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="p">(</span><span class="n">float_inp2</span><span class="p">)</span>
<span class="n">eval_mode_cat</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cat</span><span class="p">([</span><span class="n">eval_quant_inp1</span><span class="p">,</span> <span class="n">eval_quant_inp2</span><span class="p">],</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Eval mode concat quant inputs:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">eval_quant_inp1</span><span class="si">}</span><span class="s2"> </span><span class="si">{</span><span class="n">eval_quant_inp2</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Eval mode concat quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">eval_mode_cat</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Eval mode concat quant inputs:
 QuantTensor(value=tensor([[ 1.5335, -0.2875],
        [-2.0447,  0.5751],
        [-1.0863, -1.4057]]), scale=tensor(0.0160), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(False)) QuantTensor(value=tensor([[ 0.3994,  0.8307],
        [-0.7188, -0.3994],
        [-0.5910,  0.1757]]), scale=tensor(0.0160), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(False))

Eval mode concat quant output:
 QuantTensor(value=tensor([[ 1.5335, -0.2875,  0.3994,  0.8307],
        [-2.0447,  0.5751, -0.7188, -0.3994],
        [-1.0863, -1.4057, -0.5910,  0.1757]]), scale=tensor(0.0160), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(True), training_t=tensor(False))
</pre></div></div>
</div>
</section>
</section>
</section>
<section id="Customizing-Quantizers">
<h1>Customizing Quantizers<a class="headerlink" href="#Customizing-Quantizers" title="Permalink to this heading">#</a></h1>
<section id="Common-keyword-arguments">
<h2>Common keyword arguments<a class="headerlink" href="#Common-keyword-arguments" title="Permalink to this heading">#</a></h2>
<p>The easiest way to customize a quantizer is by passing appropriate keyword arguments. What happens any time we are setting a keyword argument is that we are overriding an hyperparameter in the underlying quantizer. Different quantizers expose different hyperparameters, which is why quantized layers like QuantLinear accept arbitrary <code class="docutils literal notranslate"><span class="pre">**kwargs</span></code>:</p>
<section id="Weight-bit-width">
<h3>Weight bit-width<a class="headerlink" href="#Weight-bit-width" title="Permalink to this heading">#</a></h3>
<p>Let’s look at customizing the default quantizers first. To override the <code class="docutils literal notranslate"><span class="pre">bit_width</span></code> of weights in the default <code class="docutils literal notranslate"><span class="pre">weight_quant</span></code> quantizer we set <code class="docutils literal notranslate"><span class="pre">weight_bit_width</span></code>:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[19]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">weight_bit_width</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Weight QuantTensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Weight QuantTensor:
 QuantTensor(value=tensor([[-0.0000,  0.3880],
        [-0.5820, -0.5044],
        [-0.2716,  0.1940],
        [-0.0000,  0.5432]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0388, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(5.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
</section>
<section id="Per-channel-weight-quantization">
<h3>Per-channel weight quantization<a class="headerlink" href="#Per-channel-weight-quantization" title="Permalink to this heading">#</a></h3>
<p>We can enable channel-wise quantization by setting <code class="docutils literal notranslate"><span class="pre">weight_scaling_per_output_channel=True</span></code>:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[20]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">weight_bit_width</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">weight_scaling_per_output_channel</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Weight QuantTensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Weight QuantTensor:
 QuantTensor(value=tensor([[-0.0000,  0.3793],
        [-0.5820, -0.5044],
        [-0.2723,  0.1816],
        [-0.0000,  0.5607]], grad_fn=&lt;MulBackward0&gt;), scale=tensor([[0.0253],
        [0.0388],
        [0.0182],
        [0.0374]], grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(5.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
</section>
<section id="Activation-bit-width">
<h3>Activation bit-width<a class="headerlink" href="#Activation-bit-width" title="Permalink to this heading">#</a></h3>
<p>Similarly we can set the <code class="docutils literal notranslate"><span class="pre">bit_width</span></code> for activations:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[21]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_identity</span> <span class="o">=</span> <span class="n">QuantIdentity</span><span class="p">(</span><span class="n">bit_width</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;QuantTensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_identity</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
QuantTensor:
 QuantTensor(value=tensor([[ 1.6341, -0.5447],
        [-2.1788,  0.5447],
        [-1.0894, -1.6341]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.5447, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(3.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
</section>
<section id="Activation-quantization-with-max_val-init">
<h3>Activation quantization with max_val init<a class="headerlink" href="#Activation-quantization-with-max_val-init" title="Permalink to this heading">#</a></h3>
<p>Certain quantizers require the user to pass in additional keyword arguments. This is the case for example with <code class="docutils literal notranslate"><span class="pre">Uint8ActPerTensorFloatMaxInit</span></code>, where the learned scale factor is initialized by an user-defined maximum value rather than from statistics:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[22]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">Uint8ActPerTensorFloatMaxInit</span>

<span class="n">float_inp1</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_relu</span> <span class="o">=</span> <span class="n">QuantReLU</span><span class="p">(</span><span class="n">max_val</span><span class="o">=</span><span class="mf">6.0</span><span class="p">,</span> <span class="n">act_quant</span><span class="o">=</span><span class="n">Uint8ActPerTensorFloatMaxInit</span><span class="p">,</span> <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="n">quant_relu</span><span class="p">(</span><span class="n">float_inp1</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[22]:
</pre></div>
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
QuantTensor(value=tensor([[1.5294, 0.0000],
        [0.0000, 0.5647],
        [0.0000, 0.0000]], grad_fn=&lt;MulBackward0&gt;), scale=tensor(0.0235, grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8.), signed_t=tensor(False), training_t=tensor(True))
</pre></div></div>
</div>
</section>
<section id="Per-channel-activation-quantization">
<h3>Per-channel activation quantization<a class="headerlink" href="#Per-channel-activation-quantization" title="Permalink to this heading">#</a></h3>
<p>And its also possible to perform <em>per-channel activation quantization</em> if the number of channels is statically defined by providing some extra information. This makes sense for example as input to a depthwise separable layer, like a depthwise-separable convolution. Example:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[23]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.nn</span><span class="w"> </span><span class="kn">import</span> <span class="n">QuantConv1d</span>

<span class="n">BATCHES</span> <span class="o">=</span> <span class="mi">1</span>
<span class="n">CHANNELS</span> <span class="o">=</span> <span class="mi">2</span>
<span class="n">FEATURES</span> <span class="o">=</span> <span class="mi">5</span>
<span class="n">KERNEL</span> <span class="o">=</span> <span class="mi">3</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">BATCHES</span><span class="p">,</span> <span class="n">CHANNELS</span><span class="p">,</span> <span class="n">FEATURES</span><span class="p">)</span>
<span class="n">per_channel_depthwise_quant_conv</span> <span class="o">=</span> <span class="n">QuantConv1d</span><span class="p">(</span>
    <span class="n">CHANNELS</span><span class="p">,</span> <span class="n">CHANNELS</span><span class="p">,</span> <span class="n">KERNEL</span><span class="p">,</span> <span class="n">groups</span><span class="o">=</span><span class="n">CHANNELS</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
    <span class="c1"># set the quantizers</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloat</span><span class="p">,</span>
    <span class="n">bias_quant</span><span class="o">=</span><span class="n">Int16Bias</span><span class="p">,</span>
    <span class="c1"># customize the input quantizer with extra keyword args</span>
    <span class="n">input_bit_width</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
    <span class="n">input_scaling_per_output_channel</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
    <span class="n">input_scaling_stats_permute_dims</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">),</span> <span class="c1"># permute dims to put channels first during stats collection phase</span>
    <span class="n">input_per_channel_broadcastable_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">CHANNELS</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="c1"># shape of the learned parameter</span>
    <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_output</span> <span class="o">=</span> <span class="n">per_channel_depthwise_quant_conv</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Float input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">float_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Per-channel quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Float input:
 tensor([[[ 1.5410, -0.2934, -2.1788,  0.5684, -1.0845],
         [-1.3986,  0.4033,  0.8380, -0.7193, -0.4033]]])

Per-channel quant output:
 QuantTensor(value=tensor([[[ 0.8616, -0.7012,  0.4503],
         [-1.1285, -0.4937, -0.1901]]], grad_fn=&lt;SqueezeBackward1&gt;), scale=tensor([[[0.0021],
         [0.0013]]], grad_fn=&lt;MulBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
</section>
</section>
<section id="Inheriting-from-a-quantizer">
<h2>Inheriting from a quantizer<a class="headerlink" href="#Inheriting-from-a-quantizer" title="Permalink to this heading">#</a></h2>
<p>What happens whenever we are passing a keyword argument (with prefix <code class="docutils literal notranslate"><span class="pre">weight_</span></code>, <code class="docutils literal notranslate"><span class="pre">input_</span></code>, <code class="docutils literal notranslate"><span class="pre">output_</span></code>, <code class="docutils literal notranslate"><span class="pre">bias_</span></code> for a layer like <code class="docutils literal notranslate"><span class="pre">QuantLinear</span></code>, no prefix for an activation layer like <code class="docutils literal notranslate"><span class="pre">QuantReLU</span></code>) is that we are overriding attributes of the underlying quantizer. To make things more compact and re-usable, we can also simply define a new quantizer by inheriting from the one we are customizing. The previous example then would look like this:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[24]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.nn</span><span class="w"> </span><span class="kn">import</span> <span class="n">QuantConv1d</span>

<span class="n">BATCHES</span> <span class="o">=</span> <span class="mi">1</span>
<span class="n">CHANNELS</span> <span class="o">=</span> <span class="mi">2</span>
<span class="n">FEATURES</span> <span class="o">=</span> <span class="mi">5</span>
<span class="n">KERNEL</span> <span class="o">=</span> <span class="mi">3</span>

<span class="k">class</span><span class="w"> </span><span class="nc">PerChannel3bActQuant</span><span class="p">(</span><span class="n">Int8ActPerTensorFloat</span><span class="p">):</span>
    <span class="n">bit_width</span> <span class="o">=</span> <span class="mi">3</span>
    <span class="n">scaling_per_output_channel</span><span class="o">=</span><span class="kc">True</span>
    <span class="n">scaling_stats_permute_dims</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>

<span class="n">float_input</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">BATCHES</span><span class="p">,</span> <span class="n">CHANNELS</span><span class="p">,</span> <span class="n">FEATURES</span><span class="p">)</span>
<span class="n">per_channel_depthwise_quant_conv</span> <span class="o">=</span> <span class="n">QuantConv1d</span><span class="p">(</span>
    <span class="n">CHANNELS</span><span class="p">,</span> <span class="n">CHANNELS</span><span class="p">,</span> <span class="n">KERNEL</span><span class="p">,</span> <span class="n">groups</span><span class="o">=</span><span class="n">CHANNELS</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
    <span class="c1"># set the quantizers</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">PerChannel3bActQuant</span><span class="p">,</span>
    <span class="n">bias_quant</span><span class="o">=</span><span class="n">Int16Bias</span><span class="p">,</span>
    <span class="c1"># layer-specific kwarg</span>
    <span class="n">input_per_channel_broadcastable_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">CHANNELS</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span>
    <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_output</span> <span class="o">=</span> <span class="n">per_channel_depthwise_quant_conv</span><span class="p">(</span><span class="n">float_input</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Float input:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">float_input</span><span class="si">}</span><span class="s2"> </span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Per-channel quant output:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_output</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Float input:
 tensor([[[ 1.5410, -0.2934, -2.1788,  0.5684, -1.0845],
         [-1.3986,  0.4033,  0.8380, -0.7193, -0.4033]]])

Per-channel quant output:
 QuantTensor(value=tensor([[[ 0.8616, -0.7012,  0.4503],
         [-1.1285, -0.4937, -0.1901]]], grad_fn=&lt;SqueezeBackward1&gt;), scale=tensor([[[0.0021],
         [0.0013]]], grad_fn=&lt;MulBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(17.), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
</section>
</section>
<section id="Defining-a-quantizer-from-scratch-with-enums">
<h1>Defining a quantizer from scratch with enums<a class="headerlink" href="#Defining-a-quantizer-from-scratch-with-enums" title="Permalink to this heading">#</a></h1>
<p>So far we have seen some options to customize the behaviour of existing quantizers. Let’s look now at how exactly they are defined.</p>
<p>Below is how two of the quantizers we have seen so far are declared within Brevitas using the enum-driven API (except for zero-point). As we will see in a second, it’s not the only way to define a quantizer, but it’s a way for users to easily experiment with different built-in options.</p>
<p>The quantizer defines various directives and hyperparameters driving the underlying algorithms as class level attributes.</p>
<section id="Weight-quantizer">
<h2>Weight quantizer<a class="headerlink" href="#Weight-quantizer" title="Permalink to this heading">#</a></h2>
<p>For weights, What we are doing by inheriting from <code class="docutils literal notranslate"><span class="pre">WeightQuantSolver</span></code>, meaning those directive should be translated to an implementation that makes sense for weights.</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[25]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.inject.enum</span><span class="w"> </span><span class="kn">import</span> <span class="o">*</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.zero_point</span><span class="w"> </span><span class="kn">import</span> <span class="n">ZeroZeroPoint</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant.solver</span><span class="w"> </span><span class="kn">import</span> <span class="n">WeightQuantSolver</span><span class="p">,</span> <span class="n">ActQuantSolver</span>

<span class="k">class</span><span class="w"> </span><span class="nc">Int8WeightPerTensorFloat</span><span class="p">(</span><span class="n">WeightQuantSolver</span><span class="p">):</span>
    <span class="n">quant_type</span> <span class="o">=</span> <span class="n">QuantType</span><span class="o">.</span><span class="n">INT</span> <span class="c1"># integer quantization</span>
    <span class="n">bit_width_impl_type</span> <span class="o">=</span> <span class="n">BitWidthImplType</span><span class="o">.</span><span class="n">CONST</span> <span class="c1"># constant bit width</span>
    <span class="n">float_to_int_impl_type</span> <span class="o">=</span> <span class="n">FloatToIntImplType</span><span class="o">.</span><span class="n">ROUND</span> <span class="c1"># round to nearest</span>
    <span class="n">scaling_impl_type</span> <span class="o">=</span> <span class="n">ScalingImplType</span><span class="o">.</span><span class="n">STATS</span> <span class="c1"># scale based on statistics</span>
    <span class="n">scaling_stats_op</span> <span class="o">=</span> <span class="n">StatsOp</span><span class="o">.</span><span class="n">MAX</span> <span class="c1"># scale statistics is the absmax value</span>
    <span class="n">restrict_scaling_type</span> <span class="o">=</span> <span class="n">RestrictValueType</span><span class="o">.</span><span class="n">FP</span> <span class="c1"># scale factor is a floating point value</span>
    <span class="n">scaling_per_output_channel</span> <span class="o">=</span> <span class="kc">False</span> <span class="c1"># scale is per tensor</span>
    <span class="n">bit_width</span> <span class="o">=</span> <span class="mi">8</span> <span class="c1"># bit width is 8</span>
    <span class="n">signed</span> <span class="o">=</span> <span class="kc">True</span> <span class="c1"># quantization range is signed</span>
    <span class="n">narrow_range</span> <span class="o">=</span> <span class="kc">True</span> <span class="c1"># quantization range is [-127,127] rather than [-128, 127]</span>
    <span class="n">zero_point_impl</span> <span class="o">=</span> <span class="n">ZeroZeroPoint</span> <span class="c1"># zero point is 0.</span>
</pre></div>
</div>
</div>
</section>
<section id="Activation-quantizer">
<h2>Activation quantizer<a class="headerlink" href="#Activation-quantizer" title="Permalink to this heading">#</a></h2>
<p>Similarly for activations we can inherit from <code class="docutils literal notranslate"><span class="pre">ActQuantSolver</span></code>. The generated implementation is gonna be a bit different, since we are quantizing tensors defined at runtime rather than a parameter, by it’s driven by the same enums:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[26]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="k">class</span><span class="w"> </span><span class="nc">Int8ActPerTensorFloat</span><span class="p">(</span><span class="n">ActQuantSolver</span><span class="p">):</span>
    <span class="n">quant_type</span> <span class="o">=</span> <span class="n">QuantType</span><span class="o">.</span><span class="n">INT</span> <span class="c1"># integer quantization</span>
    <span class="n">bit_width_impl_type</span> <span class="o">=</span> <span class="n">BitWidthImplType</span><span class="o">.</span><span class="n">CONST</span> <span class="c1"># constant bit width</span>
    <span class="n">float_to_int_impl_type</span> <span class="o">=</span> <span class="n">FloatToIntImplType</span><span class="o">.</span><span class="n">ROUND</span> <span class="c1"># round to nearest</span>
    <span class="n">scaling_impl_type</span> <span class="o">=</span> <span class="n">ScalingImplType</span><span class="o">.</span><span class="n">PARAMETER_FROM_STATS</span> <span class="c1"># scale is a parameter initialized from statistics</span>
    <span class="n">scaling_stats_op</span> <span class="o">=</span> <span class="n">StatsOp</span><span class="o">.</span><span class="n">PERCENTILE</span> <span class="c1"># scale statistics is a percentile of the abs value</span>
    <span class="n">high_percentile_q</span> <span class="o">=</span> <span class="mf">99.999</span> <span class="c1"># percentile is 99.999</span>
    <span class="n">collect_stats_steps</span> <span class="o">=</span> <span class="mi">300</span>  <span class="c1"># statistics are collected for 300 forward steps before switching to a learned parameter</span>
    <span class="n">restrict_scaling_type</span> <span class="o">=</span> <span class="n">RestrictValueType</span><span class="o">.</span><span class="n">FP</span> <span class="c1"># scale is a floating-point value</span>
    <span class="n">scaling_per_output_channel</span> <span class="o">=</span> <span class="kc">False</span>  <span class="c1"># scale is per tensor</span>
    <span class="n">bit_width</span> <span class="o">=</span> <span class="mi">8</span>  <span class="c1"># bit width is 8</span>
    <span class="n">signed</span> <span class="o">=</span> <span class="kc">True</span> <span class="c1"># quantization range is signed</span>
    <span class="n">narrow_range</span> <span class="o">=</span> <span class="kc">False</span> <span class="c1"># quantization range is [-128, 127] rather than [-127, 127]</span>
    <span class="n">zero_point_impl</span> <span class="o">=</span> <span class="n">ZeroZeroPoint</span> <span class="c1"># zero point is 0.</span>
</pre></div>
</div>
</div>
<p>Any of attributes in the quantizers above can be passed in or overriden as a keyword argument to the layer the quantizer is passed to (together with its appropriate prefix), with keyword arguments always having the priority over the value defined in the quantizer.</p>
</section>
<section id="Learned-scale-and-bit-width-quantizer">
<h2>Learned scale and bit-width quantizer<a class="headerlink" href="#Learned-scale-and-bit-width-quantizer" title="Permalink to this heading">#</a></h2>
<p>Let’s now look at how we can build something more advanced by tweaking a few options. We want a weight quantizer with: - per-channel scale factors learned in log domain as a parameter initialized from absmax statistics. - bit-width initialized to 8b and learned as a parameter from there.</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[27]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">Int8WeightPerTensorFloat</span>

<span class="k">class</span><span class="w"> </span><span class="nc">LearnedIntWeightPerChannelFloat</span><span class="p">(</span><span class="n">Int8WeightPerTensorFloat</span><span class="p">):</span>
    <span class="n">scaling_per_output_channel</span> <span class="o">=</span> <span class="kc">True</span>
    <span class="n">scaling_impl_type</span> <span class="o">=</span> <span class="n">ScalingImplType</span><span class="o">.</span><span class="n">PARAMETER_FROM_STATS</span>
    <span class="n">restrict_scaling_type</span> <span class="o">=</span> <span class="n">RestrictValueType</span><span class="o">.</span><span class="n">LOG_FP</span>
    <span class="n">bit_width_impl_type</span> <span class="o">=</span> <span class="n">BitWidthImplType</span><span class="o">.</span><span class="n">PARAMETER</span>


<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">weight_quant</span><span class="o">=</span><span class="n">LearnedIntWeightPerChannelFloat</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Weight QuantTensor:</span><span class="se">\n</span><span class="s2"> </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Weight QuantTensor:
 QuantTensor(value=tensor([[-0.0060,  0.3793],
        [-0.5820, -0.5224],
        [-0.2723,  0.1887],
        [-0.0132,  0.5607]], grad_fn=&lt;MulBackward0&gt;), scale=tensor([[0.0030],
        [0.0046],
        [0.0021],
        [0.0044]], grad_fn=&lt;DivBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(8., grad_fn=&lt;RoundSteFnBackward&gt;), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
<p>Notice how the quantized weights’ <code class="docutils literal notranslate"><span class="pre">bit_width</span></code> is now a value we can backpropagate through, as it exposes a <code class="docutils literal notranslate"><span class="pre">grad_fn</span></code> function. This way we can use it as part of a loss regularization function to model a particular hardware cost function, e.g. pushing larger layers to have a smaller bit width. The same principle applies to activation, and we can combine them:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[28]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="k">class</span><span class="w"> </span><span class="nc">LearnedIntActPerTensorFloat</span><span class="p">(</span><span class="n">Int8ActPerTensorFloat</span><span class="p">):</span>
    <span class="n">bit_width_impl_type</span> <span class="o">=</span> <span class="n">BitWidthImplType</span><span class="o">.</span><span class="n">PARAMETER</span>
    <span class="n">restrict_scaling_type</span> <span class="o">=</span> <span class="n">RestrictValueType</span><span class="o">.</span><span class="n">LOG_FP</span>

<span class="n">float_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">LearnedIntActPerTensorFloat</span><span class="p">,</span>
    <span class="n">weight_quant</span><span class="o">=</span><span class="n">LearnedIntWeightPerChannelFloat</span><span class="p">,</span>
    <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="n">quant_linear</span><span class="p">(</span><span class="n">float_inp</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[28]:
</pre></div>
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
QuantTensor(value=tensor([[-0.9109, -0.4588,  0.3119, -0.6530],
        [ 1.2089,  0.6493, -0.3731,  0.8706],
        [ 1.3893,  0.2823, -0.8979,  0.9543]], grad_fn=&lt;MmBackward&gt;), scale=tensor([[9.0542e-05, 3.9068e-05, 5.6866e-05, 6.4251e-05]],
       grad_fn=&lt;MulBackward0&gt;), zero_point=tensor(0.), bit_width=tensor(17., grad_fn=&lt;CeilSteFnBackward&gt;), signed_t=tensor(True), training_t=tensor(True))
</pre></div></div>
</div>
<p>As we can see, we can backpropagate through the output <code class="docutils literal notranslate"><span class="pre">bit_width</span></code>. By including it in a loss function, we could then try to control the size of the output accumulator, and in turn the <code class="docutils literal notranslate"><span class="pre">bit_width</span></code> of both weights and input.</p>
</section>
</section>
<section id="Retraining-from-floating-point">
<h1>Retraining from floating-point<a class="headerlink" href="#Retraining-from-floating-point" title="Permalink to this heading">#</a></h1>
<p>In many scenarios it’s convenient to perform quantization-aware training starting from a floating-point model. Say for example we want to load a pretrained-floating point state on top of the layer with learned bit-width and scale we just saw. We simulate it with a separate floating-point <code class="docutils literal notranslate"><span class="pre">nn.Linear</span></code>. If we didn’t do anything else we would get an error:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[29]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">torch</span><span class="w"> </span><span class="kn">import</span> <span class="n">nn</span>

<span class="n">float_linear</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">LearnedIntActPerTensorFloat</span><span class="p">,</span>
    <span class="n">weight_quant</span><span class="o">=</span><span class="n">LearnedIntWeightPerChannelFloat</span><span class="p">,</span>
    <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="n">quant_linear</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">float_linear</span><span class="o">.</span><span class="n">state_dict</span><span class="p">())</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
<span class="ansi-red-intense-fg ansi-bold">---------------------------------------------------------------------------</span>
<span class="ansi-red-intense-fg ansi-bold">RuntimeError</span>                              Traceback (most recent call last)
<span class="ansi-green-intense-fg ansi-bold">C:\Users\ALESSA~1\AppData\Local\Temp/ipykernel_18920/1653109852.py</span> in <span class="ansi-cyan-fg">&lt;module&gt;</span>
<span class="ansi-green-fg">     10</span>     return_quant_tensor=True, bias=False)
<span class="ansi-green-fg">     11</span>
<span class="ansi-green-intense-fg ansi-bold">---&gt; 12</span><span class="ansi-yellow-intense-fg ansi-bold"> </span>quant_linear<span class="ansi-yellow-intense-fg ansi-bold">.</span>load_state_dict<span class="ansi-yellow-intense-fg ansi-bold">(</span>float_linear<span class="ansi-yellow-intense-fg ansi-bold">.</span>state_dict<span class="ansi-yellow-intense-fg ansi-bold">(</span><span class="ansi-yellow-intense-fg ansi-bold">)</span><span class="ansi-yellow-intense-fg ansi-bold">)</span>

<span class="ansi-green-intense-fg ansi-bold">~\miniconda3\envs\pt190\lib\site-packages\torch\nn\modules\module.py</span> in <span class="ansi-cyan-fg">load_state_dict</span><span class="ansi-blue-intense-fg ansi-bold">(self, state_dict, strict)</span>
<span class="ansi-green-fg">   1405</span>         <span class="ansi-green-intense-fg ansi-bold">if</span> len<span class="ansi-yellow-intense-fg ansi-bold">(</span>error_msgs<span class="ansi-yellow-intense-fg ansi-bold">)</span> <span class="ansi-yellow-intense-fg ansi-bold">&gt;</span> <span class="ansi-cyan-intense-fg ansi-bold">0</span><span class="ansi-yellow-intense-fg ansi-bold">:</span>
<span class="ansi-green-fg">   1406</span>             raise RuntimeError(&#39;Error(s) in loading state_dict for {}:\n\t{}&#39;.format(
<span class="ansi-green-intense-fg ansi-bold">-&gt; 1407</span><span class="ansi-yellow-intense-fg ansi-bold">                                self.__class__.__name__, &#34;\n\t&#34;.join(error_msgs)))
</span><span class="ansi-green-fg">   1408</span>         <span class="ansi-green-intense-fg ansi-bold">return</span> _IncompatibleKeys<span class="ansi-yellow-intense-fg ansi-bold">(</span>missing_keys<span class="ansi-yellow-intense-fg ansi-bold">,</span> unexpected_keys<span class="ansi-yellow-intense-fg ansi-bold">)</span>
<span class="ansi-green-fg">   1409</span>

<span class="ansi-red-intense-fg ansi-bold">RuntimeError</span>: Error(s) in loading state_dict for QuantLinear:
        Missing key(s) in state_dict: &#34;input_quant.fused_activation_quant_proxy.tensor_quant.scaling_impl.value&#34;, &#34;input_quant.fused_activation_quant_proxy.tensor_quant.msb_clamp_bit_width_impl.bit_width_offset&#34;, &#34;weight_quant.tensor_quant.scaling_impl.value&#34;, &#34;weight_quant.tensor_quant.msb_clamp_bit_width_impl.bit_width_offset&#34;.
</pre></div></div>
</div>
<p>That’s because the quantizers for both weights and input are introducing new learned parameters that are not present in the original floating-point layer.</p>
<p>We can supress the error to accomodate the re-training scenario by either setting the env variable <code class="docutils literal notranslate"><span class="pre">BREVITAS_IGNORE_MISSING_KEYS=1</span></code>, or by enabling the corresponding config flag:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[30]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">torch</span><span class="w"> </span><span class="kn">import</span> <span class="n">nn</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>

<span class="n">config</span><span class="o">.</span><span class="n">IGNORE_MISSING_KEYS</span> <span class="o">=</span> <span class="kc">True</span>

<span class="n">float_linear</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">LearnedIntActPerTensorFloat</span><span class="p">,</span>
    <span class="n">weight_quant</span><span class="o">=</span><span class="n">LearnedIntWeightPerChannelFloat</span><span class="p">,</span>
    <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="n">quant_linear</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">float_linear</span><span class="o">.</span><span class="n">state_dict</span><span class="p">())</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[30]:
</pre></div>
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
&lt;All keys matched successfully&gt;
</pre></div></div>
</div>
<p>However, in this scenario we are saying we want the learned component of the scale factor for weights to be initialized to the maximum value of the weight tensor on a per channel basis. So even though it’s not part of the floating-point state dict, the value of the weight scale factor should be updated. Thanks to how the weight quantizer works this happens <em>automatically</em>:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[31]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">torch</span><span class="w"> </span><span class="kn">import</span> <span class="n">nn</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas</span><span class="w"> </span><span class="kn">import</span> <span class="n">config</span>

<span class="n">config</span><span class="o">.</span><span class="n">IGNORE_MISSING_KEYS</span> <span class="o">=</span> <span class="kc">True</span>

<span class="n">float_linear</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">LearnedIntActPerTensorFloat</span><span class="p">,</span>
    <span class="n">weight_quant</span><span class="o">=</span><span class="n">LearnedIntWeightPerChannelFloat</span><span class="p">,</span>
    <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Weight scale before load_state_dict: </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="o">.</span><span class="n">scale</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>

<span class="n">quant_linear</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">float_linear</span><span class="o">.</span><span class="n">state_dict</span><span class="p">())</span>

<span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Weight scale after load_state_dict: </span><span class="si">{</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">quant_weight</span><span class="p">()</span><span class="o">.</span><span class="n">scale</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Weight scale before load_state_dict: tensor([[0.0015],
        [0.0017],
        [0.0053],
        [0.0023]], grad_fn=&lt;DivBackward0&gt;)
Weight scale after load_state_dict: tensor([[0.0030],
        [0.0046],
        [0.0021],
        [0.0044]], grad_fn=&lt;DivBackward0&gt;)
</pre></div></div>
</div>
<p>This works also in the scenario where the weight quantizer is shared among multiple layers.</p>
</section>
<section id="Defining-a-quantizer-from-scratch-with-dependency-injection">
<h1>Defining a quantizer from scratch with dependency-injection<a class="headerlink" href="#Defining-a-quantizer-from-scratch-with-dependency-injection" title="Permalink to this heading">#</a></h1>
<p>So far we have seen how users can pick and adopt various quantization algorithms that can be expressed through components already built into Brevitas. The more interesting question is how to expressed completely novel algorithms based on new quantization components.</p>
<section id="Activation-quantization-from-scratch">
<h2>Activation quantization from scratch<a class="headerlink" href="#Activation-quantization-from-scratch" title="Permalink to this heading">#</a></h2>
<p>To do so we need to move away from the enum-driven API, and understand how quantizers work underneath. If we look at the structure of a layer like <code class="docutils literal notranslate"><span class="pre">QuantReLU</span></code>, we can immediately notice that it’s formed by a variety of nested modules:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[32]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">QuantReLU</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[32]:
</pre></div>
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
QuantReLU(
  (input_quant): ActQuantProxyFromInjector(
    (_zero_hw_sentinel): StatelessBuffer()
  )
  (act_quant): ActQuantProxyFromInjector(
    (_zero_hw_sentinel): StatelessBuffer()
    (fused_activation_quant_proxy): FusedActivationQuantProxy(
      (activation_impl): ReLU()
      (tensor_quant): RescalingIntQuant(
        (int_quant): IntQuant(
          (float_to_int_impl): RoundSte()
          (tensor_clamp_impl): TensorClamp()
          (delay_wrapper): DelayWrapper(
            (delay_impl): _NoDelay()
          )
        )
        (scaling_impl): ParameterFromRuntimeStatsScaling(
          (stats_input_view_shape_impl): OverTensorView()
          (stats): _Stats(
            (stats_impl): AbsPercentile()
          )
          (restrict_clamp_scaling): _RestrictClampValue(
            (clamp_min_ste): Identity()
            (restrict_value_impl): FloatRestrictValue()
          )
          (restrict_inplace_preprocess): Identity()
          (restrict_preprocess): Identity()
        )
        (int_scaling_impl): IntScaling()
        (zero_point_impl): ZeroZeroPoint(
          (zero_point): StatelessBuffer()
        )
        (msb_clamp_bit_width_impl): BitWidthConst(
          (bit_width): StatelessBuffer()
        )
      )
    )
  )
)
</pre></div></div>
</div>
<p>Where the implementation of the quantization component is expressed by the <code class="docutils literal notranslate"><span class="pre">tensor_quant</span></code> module.</p>
<p>The idea is that we express a quantization algorithm as various <code class="docutils literal notranslate"><span class="pre">nn.Module</span></code> combined in a depedency-injection fashion, meaning that they combine from the outside-in through standardized interfaces.</p>
<p>So we have different modules to express a particular type of scale factor, a particular type of rounding, a particular integer quantization procedure, and so on. The overall quantization algorithm is then expressed as a nested assembly of all its different pieces. If we were to do it manually for the layer above, it would look like this:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[33]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.function_wrapper</span><span class="w"> </span><span class="kn">import</span> <span class="n">TensorClamp</span><span class="p">,</span> <span class="n">OverTensorView</span><span class="p">,</span> <span class="n">RoundSte</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.scaling</span><span class="w"> </span><span class="kn">import</span> <span class="n">ParameterFromRuntimeStatsScaling</span><span class="p">,</span> <span class="n">IntScaling</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.stats</span><span class="w"> </span><span class="kn">import</span> <span class="n">AbsPercentile</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.restrict_val</span><span class="w"> </span><span class="kn">import</span> <span class="n">FloatRestrictValue</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.bit_width</span><span class="w"> </span><span class="kn">import</span> <span class="n">BitWidthConst</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">IntQuant</span><span class="p">,</span> <span class="n">RescalingIntQuant</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.zero_point</span><span class="w"> </span><span class="kn">import</span> <span class="n">ZeroZeroPoint</span>

<span class="n">tensor_quant</span> <span class="o">=</span> <span class="n">RescalingIntQuant</span><span class="p">(</span>
    <span class="n">int_quant</span><span class="o">=</span><span class="n">IntQuant</span><span class="p">(</span>
        <span class="n">float_to_int_impl</span><span class="o">=</span><span class="n">RoundSte</span><span class="p">(),</span>
        <span class="n">tensor_clamp_impl</span><span class="o">=</span><span class="n">TensorClamp</span><span class="p">(),</span>
        <span class="n">signed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
        <span class="n">narrow_range</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
    <span class="n">zero_point_impl</span><span class="o">=</span><span class="n">ZeroZeroPoint</span><span class="p">(),</span>
    <span class="n">bit_width_impl</span><span class="o">=</span><span class="n">BitWidthConst</span><span class="p">(</span><span class="n">bit_width</span><span class="o">=</span><span class="mi">8</span><span class="p">),</span>
    <span class="n">int_scaling_impl</span><span class="o">=</span><span class="n">IntScaling</span><span class="p">(</span><span class="n">signed</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">narrow_range</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
    <span class="n">scaling_impl</span><span class="o">=</span><span class="n">ParameterFromRuntimeStatsScaling</span><span class="p">(</span>
        <span class="n">scaling_stats_input_view_shape_impl</span><span class="o">=</span><span class="n">OverTensorView</span><span class="p">(),</span>
        <span class="n">scaling_stats_impl</span><span class="o">=</span><span class="n">AbsPercentile</span><span class="p">(</span>
            <span class="n">high_percentile_q</span><span class="o">=</span><span class="mf">99.999</span><span class="p">,</span> <span class="n">stats_reduce_dim</span><span class="o">=</span><span class="kc">None</span><span class="p">),</span>
        <span class="n">collect_stats_steps</span><span class="o">=</span><span class="mi">300</span><span class="p">)</span>
<span class="p">)</span>
</pre></div>
</div>
</div>
<p>With this approach we can express arbitrary quantization algorithms in a very modular fashion. However, we have two extra constraints we care about:</p>
<ul class="simple">
<li><p>This degree of modularity comes at a cost, i.e. the overhead of doing a lot of small operations scattered across different modules in Python shows. We want to avoid it by taking advantage of PyTorch’s TorchScript JIT compiler, meaning the <code class="docutils literal notranslate"><span class="pre">tensor_quant</span></code> module above should be fused and end-to-end compiled.</p></li>
<li><p>We want a way to define our quantizer such that it can be re-instantiated and re-compiled on-demand in certain scenarios, like whenever it’s shared among multiple layers or when a pretrained <code class="docutils literal notranslate"><span class="pre">state_dict</span></code> is load on top of it.</p></li>
</ul>
<p>To solve the first issue, we restrict ourselves to using only <code class="docutils literal notranslate"><span class="pre">ScriptModule</span></code>, a variant of a <code class="docutils literal notranslate"><span class="pre">nn.Module</span></code> that can be JIT-compiled. This is also why we favour composition over inheritance, as TorchScript has almost no support for inheritance. So all the modules we are instantiating above derive from an alias to a ScriptModule called <code class="docutils literal notranslate"><span class="pre">brevitas.jit.ScriptModule</span></code>, whose compilation can be enabled by setting the env variable <code class="docutils literal notranslate"><span class="pre">BREVITAS_JIT=1</span></code>.</p>
<p>To solve the second issue, we adopt and extend an auto-wiring dependency injection library called <a class="reference external" href="https://github.com/proofit404/dependencies">dependencies</a> that performs the assembly and instantiation process of <code class="docutils literal notranslate"><span class="pre">tensor_quant</span></code> automatically. Let’s look at an example to make things more clear:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[34]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.inject</span><span class="w"> </span><span class="kn">import</span> <span class="n">ExtendedInjector</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.proxy</span><span class="w"> </span><span class="kn">import</span> <span class="n">ActQuantProxyFromInjector</span>

<span class="k">class</span><span class="w"> </span><span class="nc">Int8ActPerTensorFloatFromScratch</span><span class="p">(</span><span class="n">ExtendedInjector</span><span class="p">):</span>
    <span class="n">proxy_class</span> <span class="o">=</span> <span class="n">ActQuantProxyFromInjector</span> <span class="c1"># Python wrapper, returns QuantTensor</span>
    <span class="n">tensor_quant</span> <span class="o">=</span> <span class="n">RescalingIntQuant</span> <span class="c1"># TorchScript implementation, returns (dequant_value, scale, zero_point, bit_width)</span>
    <span class="n">int_quant</span> <span class="o">=</span> <span class="n">IntQuant</span>
    <span class="n">float_to_int_impl</span> <span class="o">=</span> <span class="n">RoundSte</span>
    <span class="n">zero_point_impl</span> <span class="o">=</span> <span class="n">ZeroZeroPoint</span>
    <span class="n">bit_width_impl</span> <span class="o">=</span> <span class="n">BitWidthConst</span>
    <span class="n">tensor_clamp_impl</span> <span class="o">=</span> <span class="n">TensorClamp</span>
    <span class="n">int_scaling_impl</span> <span class="o">=</span> <span class="n">IntScaling</span>
    <span class="n">scaling_impl</span> <span class="o">=</span> <span class="n">ParameterFromRuntimeStatsScaling</span>
    <span class="n">scaling_stats_input_view_shape_impl</span> <span class="o">=</span> <span class="n">OverTensorView</span>
    <span class="n">scaling_stats_impl</span> <span class="o">=</span> <span class="n">AbsPercentile</span>
    <span class="n">restrict_scaling_impl</span> <span class="o">=</span> <span class="n">FloatRestrictValue</span>
    <span class="n">high_percentile_q</span> <span class="o">=</span> <span class="mf">99.999</span>
    <span class="n">collect_stats_steps</span> <span class="o">=</span> <span class="mi">300</span>
    <span class="n">scaling_shape</span> <span class="o">=</span> <span class="p">()</span>
    <span class="n">stats_reduce_dim</span> <span class="o">=</span> <span class="kc">None</span>
    <span class="n">bit_width</span> <span class="o">=</span> <span class="mi">8</span>
    <span class="n">narrow_range</span> <span class="o">=</span> <span class="kc">True</span>
    <span class="n">signed</span> <span class="o">=</span> <span class="kc">True</span>
</pre></div>
</div>
</div>
<p><code class="docutils literal notranslate"><span class="pre">Int8ActPerTensorFloatFromScratch</span></code> is equivalent to <code class="docutils literal notranslate"><span class="pre">Int8ActPerTensorFloat</span></code>, but defined directly in terms of the modules that define the implementation, rather than with enums. As such, we inherit from our dependency-injection base class, the <code class="docutils literal notranslate"><span class="pre">ExtendedInjector</span></code>. The quantizer needs to define two attributes: - <code class="docutils literal notranslate"><span class="pre">proxy_class</span></code> represents the Python wrapper that interfaces a quantized layer like QuantIdentity with the quantizer itself, and for convinience is defined as part of the quantizer
itself. It has to return a <code class="docutils literal notranslate"><span class="pre">QuantTensor</span></code>. - <code class="docutils literal notranslate"><span class="pre">tensor_quant</span></code> is the actual ScriptModule implementating our quantization algorithm. It has to return a 4-tuple containing tensors (dequant_value, scale, zero_point, bit_width).</p>
<p><code class="docutils literal notranslate"><span class="pre">tensor_quant</span></code> is automatically instantiated by the dependency-injection engine by matching its keyword argument to attributes of the <code class="docutils literal notranslate"><span class="pre">ÈxtendedInjector</span></code>. If one of the attributes is a class that requires to be instantiated the process is repeated in a recursive manner. If it’s already an instance, it’s taken as is.</p>
<p>Note that the enum-driven API only adds an extra layer of abstraction on top of this, but it maps to the same components with the same names.</p>
<p>This way, indipendently of how it’s originally defined, a user can override any component of an existing quantizer in a fine grained manner, or define a new quantizer from scratch with entirely custom components.</p>
</section>
<section id="Weight-quantization-with-learned-scale-from-scratch">
<h2>Weight quantization with learned scale from scratch<a class="headerlink" href="#Weight-quantization-with-learned-scale-from-scratch" title="Permalink to this heading">#</a></h2>
<p>Similarly, we can define a weight quantizer from scratch with a learned scale factor implemented by <code class="docutils literal notranslate"><span class="pre">ParameterScaling</span></code> module. To initialize it, we define <code class="docutils literal notranslate"><span class="pre">scaling_init</span></code> as a function of the full precision weights to quantize, by taking the absmax of weight tensor, as with did in previous examples. To do so, we define a <code class="docutils literal notranslate"><span class="pre">&#64;value</span></code> function, a function which is evaluated at <em>dependency-injection time</em>:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[35]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.inject</span><span class="w"> </span><span class="kn">import</span> <span class="n">value</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.proxy</span><span class="w"> </span><span class="kn">import</span> <span class="n">WeightQuantProxyFromInjector</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.core.scaling</span><span class="w"> </span><span class="kn">import</span> <span class="n">ParameterScaling</span>

<span class="k">class</span><span class="w"> </span><span class="nc">Int8ActPerTensorFloatParameterFromScratch</span><span class="p">(</span><span class="n">ExtendedInjector</span><span class="p">):</span>

    <span class="nd">@value</span>
    <span class="k">def</span><span class="w"> </span><span class="nf">scaling_init</span><span class="p">(</span><span class="n">module</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">module</span><span class="o">.</span><span class="n">weight</span><span class="o">.</span><span class="n">abs</span><span class="p">()</span><span class="o">.</span><span class="n">max</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span>

    <span class="n">proxy_class</span> <span class="o">=</span> <span class="n">WeightQuantProxyFromInjector</span> <span class="c1"># Python wrapper, returns QuantTensor</span>
    <span class="n">tensor_quant</span> <span class="o">=</span> <span class="n">RescalingIntQuant</span>
    <span class="n">int_quant</span> <span class="o">=</span> <span class="n">IntQuant</span>
    <span class="n">float_to_int_impl</span> <span class="o">=</span> <span class="n">RoundSte</span>
    <span class="n">zero_point_impl</span> <span class="o">=</span> <span class="n">ZeroZeroPoint</span>
    <span class="n">bit_width_impl</span> <span class="o">=</span> <span class="n">BitWidthConst</span>
    <span class="n">tensor_clamp_impl</span> <span class="o">=</span> <span class="n">TensorClamp</span>
    <span class="n">int_scaling_impl</span> <span class="o">=</span> <span class="n">IntScaling</span>
    <span class="n">scaling_impl</span> <span class="o">=</span> <span class="n">ParameterScaling</span>
    <span class="n">restrict_scaling_impl</span> <span class="o">=</span> <span class="n">FloatRestrictValue</span>
    <span class="n">scaling_shape</span> <span class="o">=</span> <span class="p">()</span>
    <span class="n">bit_width</span> <span class="o">=</span> <span class="mi">8</span>
    <span class="n">narrow_range</span> <span class="o">=</span> <span class="kc">True</span>
    <span class="n">signed</span> <span class="o">=</span> <span class="kc">True</span>

<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">weight_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloatParameterFromScratch</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
</pre></div>
</div>
</div>
<p>This is how <code class="docutils literal notranslate"><span class="pre">scaling_impl_type</span> <span class="pre">=</span> <span class="pre">ScalingImplType.PARAMETER_FROM_STATS</span></code> for weight quantization works underneath.</p>
<p>the argument <code class="docutils literal notranslate"><span class="pre">module</span></code> of <code class="docutils literal notranslate"><span class="pre">scaling_init</span></code> represents the layer (or tuple of layers) the quantizer is passed to, and it allows the quantizer to depend on properties of the layer to quantize.</p>
<p>By declaring <code class="docutils literal notranslate"><span class="pre">scaling_init</span></code> in this way we can easily recompute it any time the layer changes. As we saw previously, for changes to the <code class="docutils literal notranslate"><span class="pre">state_dict</span></code> this happens automatically. For in-place changes like weight initialization, which cannot easily be intercepted automatically, we can invoke it manually by calling <code class="docutils literal notranslate"><span class="pre">quant_linear.weight_quant.init_tensor_quant()</span></code>.</p>
</section>
<section id="Sharing-learned-bit-width-among-layers">
<h2>Sharing learned bit-width among layers<a class="headerlink" href="#Sharing-learned-bit-width-among-layers" title="Permalink to this heading">#</a></h2>
<p>We saw in a previous section how to share entire quantizers among layers. We are now interested in sharing only certain components, and we leverage the dependency-injection mechanism we just saw to do so.</p>
<p>For example, we want the <code class="docutils literal notranslate"><span class="pre">QuantIdentity</span></code> and the <code class="docutils literal notranslate"><span class="pre">QuantLinear</span></code> with learned bit-width of our previous example to share the same learned bit-width. That can be useful whenever a particular hardware target supports only operations among tensors of the same datatype . With some verbosity, doing so is possible:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[36]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">float_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
<span class="n">quant_linear</span> <span class="o">=</span> <span class="n">QuantLinear</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="n">weight_quant</span><span class="o">=</span><span class="n">LearnedIntWeightPerChannelFloat</span><span class="p">,</span> <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="n">quant_identity</span> <span class="o">=</span> <span class="n">QuantIdentity</span><span class="p">(</span>
    <span class="n">bit_width_impl</span><span class="o">=</span><span class="n">quant_linear</span><span class="o">.</span><span class="n">weight_quant</span><span class="o">.</span><span class="n">tensor_quant</span><span class="o">.</span><span class="n">msb_clamp_bit_width_impl</span><span class="p">,</span>
    <span class="n">act_quant</span><span class="o">=</span><span class="n">LearnedIntActPerTensorFloat</span><span class="p">,</span> <span class="n">return_quant_tensor</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">quant_identity_bit_width</span> <span class="o">=</span> <span class="n">quant_identity</span><span class="o">.</span><span class="n">act_quant</span><span class="o">.</span><span class="n">fused_activation_quant_proxy</span><span class="o">.</span><span class="n">tensor_quant</span><span class="o">.</span><span class="n">msb_clamp_bit_width_impl</span>
<span class="n">quant_linear_bit_width</span> <span class="o">=</span> <span class="n">quant_linear</span><span class="o">.</span><span class="n">weight_quant</span><span class="o">.</span><span class="n">tensor_quant</span><span class="o">.</span><span class="n">msb_clamp_bit_width_impl</span>

<span class="n">quant_identity_bit_width</span> <span class="ow">is</span> <span class="n">quant_linear_bit_width</span>
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[36]:
</pre></div>
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
True
</pre></div></div>
</div>
<p>As we can see the two bit-width implementations are the same instance. What we are doing is sharing the instance of <code class="docutils literal notranslate"><span class="pre">msb_clamp_bit_width_impl</span></code> from one quantizer to another.</p>
</section>
</section>
<section id="Export">
<h1>Export<a class="headerlink" href="#Export" title="Permalink to this heading">#</a></h1>
<p>Brevitas doesn’t perform any kind of acceleration on its own. To do so, a quantized model needs to be exported to a backend first. In a perfect world there would be a single way to represent quantized models. In practice however that’s not the case, and different toolchains make different assumptions on what kind of quantization is supported and how it should be represented.</p>
<p>Brevitas then supports different flavours of export at different levels of abstractions by taking advantage of PyTorch’s support for custom symbolic representations, and in particular ONNX.</p>
<p>Existing export flows assume static quantization, i.e. scales, zero-point and bit-width need to be indipedentent from the input. All export flows abstract away from the specifics of how scales, zero-point and bit-width have been determined. However, different export flows provide supports only for certain combinations of scales, zero-point, precision, or structure of the model.</p>
<p>We install onnx and onnxoptimizer as they are required for export, and we use Netron to visualize the exported models:</p>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[37]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">!</span>pip<span class="w"> </span>install<span class="w"> </span>netron<span class="w"> </span>onnx<span class="w"> </span>onnxoptimizer
</pre></div>
</div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Requirement already satisfied: netron in c:\users\alessandro\miniconda3\envs\pt190\lib\site-packages (5.3.9)
Requirement already satisfied: onnx in c:\users\alessandro\miniconda3\envs\pt190\lib\site-packages (1.10.2)
Requirement already satisfied: onnxoptimizer in c:\users\alessandro\miniconda3\envs\pt190\lib\site-packages (0.2.6)
Requirement already satisfied: numpy&gt;=1.16.6 in c:\users\alessandro\miniconda3\envs\pt190\lib\site-packages (from onnx) (1.21.2)
Requirement already satisfied: typing-extensions&gt;=3.6.2.1 in c:\users\alessandro\miniconda3\envs\pt190\lib\site-packages (from onnx) (3.10.0.2)
Requirement already satisfied: protobuf in c:\users\alessandro\miniconda3\envs\pt190\lib\site-packages (from onnx) (3.19.1)
Requirement already satisfied: six in c:\users\alessandro\miniconda3\envs\pt190\lib\site-packages (from onnx) (1.16.0)
</pre></div></div>
</div>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[38]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">netron</span>
<span class="kn">import</span><span class="w"> </span><span class="nn">time</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">IPython.display</span><span class="w"> </span><span class="kn">import</span> <span class="n">IFrame</span>

<span class="k">def</span><span class="w"> </span><span class="nf">show_netron</span><span class="p">(</span><span class="n">model_path</span><span class="p">,</span> <span class="n">port</span><span class="p">):</span>
    <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mf">3.</span><span class="p">)</span>
    <span class="n">netron</span><span class="o">.</span><span class="n">start</span><span class="p">(</span><span class="n">model_path</span><span class="p">,</span> <span class="n">address</span><span class="o">=</span><span class="p">(</span><span class="s2">&quot;localhost&quot;</span><span class="p">,</span> <span class="n">port</span><span class="p">),</span> <span class="n">browse</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">IFrame</span><span class="p">(</span><span class="n">src</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;http://localhost:</span><span class="si">{</span><span class="n">port</span><span class="si">}</span><span class="s2">/&quot;</span><span class="p">,</span> <span class="n">width</span><span class="o">=</span><span class="s2">&quot;100%&quot;</span><span class="p">,</span> <span class="n">height</span><span class="o">=</span><span class="mi">400</span><span class="p">)</span>
</pre></div>
</div>
</div>
<section id="Export-to-ONNX-QOps">
<h2>Export to ONNX QOps<a class="headerlink" href="#Export-to-ONNX-QOps" title="Permalink to this heading">#</a></h2>
<p>Say we want to export a QuantConv1d with 4b symmetric weights, 8b symmetric inputs and outputs, and 16 biases. We can export it to a ONNX’s <code class="docutils literal notranslate"><span class="pre">QLinearConv</span></code>, but some information will be lost. In particular, weights will be represented as 8b and bias as 32b, even though they are respectively 4b and 16b. This is because ONNX does not provide a standardized way to represent them as such:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[ ]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.nn</span><span class="w"> </span><span class="kn">import</span> <span class="n">QuantConv1d</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">Int8WeightPerTensorFloat</span><span class="p">,</span> <span class="n">Int8ActPerTensorFloat</span><span class="p">,</span> <span class="n">Int16Bias</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.export</span><span class="w"> </span><span class="kn">import</span> <span class="n">export_onnx_qop</span>

<span class="n">float_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>

<span class="n">quant_conv_4b8b</span> <span class="o">=</span> <span class="n">QuantConv1d</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">weight_bit_width</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloat</span><span class="p">,</span>
    <span class="n">output_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloat</span><span class="p">,</span>
    <span class="n">bias_quant</span><span class="o">=</span><span class="n">Int16Bias</span><span class="p">)</span>

<span class="n">output_path</span> <span class="o">=</span> <span class="s1">&#39;qop_onnx_conv_4b8b.onnx&#39;</span>
<span class="n">export_onnx_qop</span><span class="p">(</span><span class="n">quant_conv_4b8b</span><span class="p">,</span> <span class="n">input_t</span><span class="o">=</span><span class="n">float_inp</span><span class="p">,</span> <span class="n">export_path</span><span class="o">=</span><span class="n">output_path</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[39]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">show_netron</span><span class="p">(</span><span class="n">output_path</span><span class="p">,</span> <span class="mi">8082</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Serving &#39;qop_onnx_conv_4b8b.onnx&#39; at http://localhost:8082
</pre></div></div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[39]:
</pre></div>
</div>
<div class="output_area rendered_html docutils container">
<iframe
    width="100%"
    height="400"
    src="http://localhost:8082/"
    frameborder="0"
    allowfullscreen

></iframe></div>
</div>
<p>In general the standard ONNX opset doesn’t support representing quantization below 8b. Additionally, ONNX QOp representation requires an output quantizer to be set at part of of the layer.</p>
<p>The constraint of always having an output quantizer is relaxed in the more recently introduced QDQ style of representation (for which there is support in Brevitas starting from version 0.8), which uses only <code class="docutils literal notranslate"><span class="pre">QuantizeLinear</span></code> and <code class="docutils literal notranslate"><span class="pre">DequantizeLinear</span></code> to represent quantization, but even with that support is still limited to 8b quantization.</p>
</section>
<section id="Export-to-custom-Quantized-ONNX">
<h2>Export to custom Quantized ONNX<a class="headerlink" href="#Export-to-custom-Quantized-ONNX" title="Permalink to this heading">#</a></h2>
<p>As an alternative, we can export it to QONNX, a custom ONNX dialect that Brevitas defines with support for custom quantization operators that can capture those informations:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[ ]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.export</span><span class="w"> </span><span class="kn">import</span> <span class="n">export_qonnx</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">Int8WeightPerTensorFloat</span><span class="p">,</span> <span class="n">Int8ActPerTensorFloat</span><span class="p">,</span> <span class="n">Int16Bias</span>

<span class="n">float_inp</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">5</span><span class="p">)</span>

<span class="n">quant_conv_4b8b</span> <span class="o">=</span> <span class="n">QuantConv1d</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">weight_bit_width</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloat</span><span class="p">,</span>
    <span class="n">output_quant</span><span class="o">=</span><span class="n">Int8ActPerTensorFloat</span><span class="p">,</span>
    <span class="n">bias_quant</span><span class="o">=</span><span class="n">Int16Bias</span><span class="p">)</span>

<span class="n">output_path</span> <span class="o">=</span> <span class="s1">&#39;brevitas_onnx_conv4b8b.onnx&#39;</span>
<span class="n">export_qonnx</span><span class="p">(</span><span class="n">quant_conv_4b8b</span><span class="p">,</span> <span class="n">input_t</span><span class="o">=</span><span class="n">float_inp</span><span class="p">,</span> <span class="n">export_path</span><span class="o">=</span><span class="n">output_path</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[40]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">show_netron</span><span class="p">(</span><span class="n">output_path</span><span class="p">,</span> <span class="mi">8083</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Serving &#39;brevitas_onnx_conv4b8b.onnx&#39; at http://localhost:8083
</pre></div></div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[40]:
</pre></div>
</div>
<div class="output_area rendered_html docutils container">
<iframe
    width="100%"
    height="400"
    src="http://localhost:8083/"
    frameborder="0"
    allowfullscreen

></iframe></div>
</div>
<p>In the <code class="docutils literal notranslate"><span class="pre">Quant</span></code> nodes above, arbitrary scale, zero-point and bit-widths are supported. This way we can support exporting scenarios where, for example, we are quantizing only weights to 4b, which a QLinearConv wouldn’t be able to capture:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[ ]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">torch</span><span class="o">.</span><span class="n">manual_seed</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>

<span class="n">quant_conv_4b_weights</span> <span class="o">=</span> <span class="n">QuantConv1d</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">weight_bit_width</span><span class="o">=</span><span class="mi">4</span><span class="p">)</span>

<span class="n">output_path</span> <span class="o">=</span> <span class="s1">&#39;brevitas_onnx_conv_4b_weights.onnx&#39;</span>
<span class="n">export_qonnx</span><span class="p">(</span><span class="n">quant_conv_4b_weights</span><span class="p">,</span> <span class="n">input_t</span><span class="o">=</span><span class="n">float_inp</span><span class="p">,</span> <span class="n">export_path</span><span class="o">=</span><span class="n">output_path</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[41]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">show_netron</span><span class="p">(</span><span class="n">output_path</span><span class="p">,</span> <span class="mi">8084</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Serving &#39;brevitas_onnx_conv_4b_weights.onnx&#39; at http://localhost:8084
</pre></div></div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[41]:
</pre></div>
</div>
<div class="output_area rendered_html docutils container">
<iframe
    width="100%"
    height="400"
    src="http://localhost:8084/"
    frameborder="0"
    allowfullscreen

></iframe></div>
</div>
<p>The custom format shown above can integrated into ONNX-based toolchains, e.g. it’s supported by our own FINN toolchain for low-precision dataflow style custom FPGAs implementations, and would be a starting point for direct integration with TVM.</p>
</section>
<section id="Export-to-TorchScript-quantization-backend">
<h2>Export to TorchScript quantization backend<a class="headerlink" href="#Export-to-TorchScript-quantization-backend" title="Permalink to this heading">#</a></h2>
<p>It’s also possible to export to TorchScript own quantized functional operators, which come with their own set of restrictions. In particular, weights should be 7b and unsigned, which requires a zero-point. We can model that with appropriate quantizers:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[ ]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.quant</span><span class="w"> </span><span class="kn">import</span> <span class="n">ShiftedUint8ActPerTensorFloat</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.export</span><span class="w"> </span><span class="kn">import</span> <span class="n">export_torch_qop</span>


<span class="n">quant_conv_8b7b</span> <span class="o">=</span> <span class="n">QuantConv1d</span><span class="p">(</span>
    <span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">bias</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
    <span class="n">input_quant</span><span class="o">=</span><span class="n">ShiftedUint8ActPerTensorFloat</span><span class="p">,</span>
    <span class="n">output_quant</span><span class="o">=</span><span class="n">ShiftedUint8ActPerTensorFloat</span><span class="p">,</span>
    <span class="n">weight_bit_width</span><span class="o">=</span><span class="mi">7</span><span class="p">,</span>
    <span class="n">bias_quant</span><span class="o">=</span><span class="n">Int16Bias</span><span class="p">)</span>

<span class="n">output_path</span> <span class="o">=</span> <span class="s1">&#39;pytorch_qf_conv_8b7b.pt&#39;</span>
<span class="n">export_torch_qop</span><span class="p">(</span><span class="n">quant_conv_8b7b</span><span class="p">,</span> <span class="n">input_t</span><span class="o">=</span><span class="n">float_inp</span><span class="p">,</span> <span class="n">export_path</span><span class="o">=</span><span class="n">output_path</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nbinput docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[42]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">show_netron</span><span class="p">(</span><span class="n">output_path</span><span class="p">,</span> <span class="mi">8085</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area stderr docutils container">
<div class="highlight"><pre>
c:\users\alessandro\documenti\brevitas_tvmcon\src\brevitas\quant_tensor\__init__.py:74: TracerWarning: torch.tensor results are registered as constants in the trace. You can safely ignore this warning if you use this function to create tensors out of constant variables that would be the same every time you call this function. In any other case, this might cause the trace to be incorrect.
  training = torch.tensor(training, dtype=torch.bool)
</pre></div></div>
</div>
<div class="nboutput docutils container">
<div class="prompt empty docutils container">
</div>
<div class="output_area docutils container">
<div class="highlight"><pre>
Serving &#39;pytorch_qf_conv_8b7b.pt&#39; at http://localhost:8085
</pre></div></div>
</div>
<div class="nboutput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[42]:
</pre></div>
</div>
<div class="output_area rendered_html docutils container">
<iframe
    width="100%"
    height="400"
    src="http://localhost:8085/"
    frameborder="0"
    allowfullscreen

></iframe></div>
</div>
<p>As we can see though information on the fact that activations are 7b is lost, and they simply marked as 8b.</p>
<p>Additionally, because bias quantization is not represented explicitly (although it is performed implicitly at 32b at runtime in the backend), any information around that is lost. As with standard ONNX, representing precisions below 8b is not possible.</p>
</section>
</section>
<section id="Brevitas-and-FX">
<h1>Brevitas and FX<a class="headerlink" href="#Brevitas-and-FX" title="Permalink to this heading">#</a></h1>
<p>In this tutorial we have covered how to incorporate quantized layers into the design of a neural networks. However, in some scenarios it might convenient to start from an existing definition of a neural networks in floating-point, and programmatically replace floating-point layers with quantized layers according to some patterns.</p>
<p>PyTorch’s recently introduced FX graph subsystem enables this kind of transformations. While covering FX goes beyond the scope of this tutorial, it’s worth mentioning that Brevitas has embraced FX, and it actually implements a backport of PyTorch’s 1.8 FX (the current LTS) under <code class="docutils literal notranslate"><span class="pre">brevitas.fx</span></code>, together with a custom input-driven tracer (<code class="docutils literal notranslate"><span class="pre">brevitas.fx.value_trace</span></code>) which similarly to <code class="docutils literal notranslate"><span class="pre">torch.jit.trace</span></code> allows to trace through conditionals and unpacking operations, as well as various graph
transformations under <code class="docutils literal notranslate"><span class="pre">brevitas.graph</span></code>.</p>
</section>
<section id="Calibration-based-post-training-quantization">
<h1>Calibration-based post-training quantization<a class="headerlink" href="#Calibration-based-post-training-quantization" title="Permalink to this heading">#</a></h1>
<p>While Brevitas was originally designed for quantization-aware training, performing calibration-based post-training quantization is possible, and it can be useful on its own or an intermediate step between floating-point training and quantization-aware retraining.</p>
<p>The idea of calibration-based quantization is to performs forward passes only with a small set of data and collect statistics to determine scale factors and zero-points.</p>
<p>Any Brevitas quantizer that is based on statistics can be used for this purpose, with the caveat that don’t want quantization to be enabled while statistics are being collected, or the data wouldn’t be representative of what the floating-point model does, so we temporarely disabling quantization while doing so. Afterwards, we re-enable calibration and apply bias correction.</p>
<p>Assuming <code class="docutils literal notranslate"><span class="pre">quant_model</span></code> is a quantized model, with pretrained floating-point weights loaded on top, we can do as follows:</p>
<div class="nbinput nblast docutils container">
<div class="prompt highlight-none notranslate"><div class="highlight"><pre><span></span>[44]:
</pre></div>
</div>
<div class="input_area highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.graph.calibrate</span><span class="w"> </span><span class="kn">import</span> <span class="n">bias_correction_mode</span>
<span class="kn">from</span><span class="w"> </span><span class="nn">brevitas.graph.calibrate</span><span class="w"> </span><span class="kn">import</span> <span class="n">calibration_mode</span>

<span class="k">def</span><span class="w"> </span><span class="nf">calibrate_model</span><span class="p">(</span><span class="n">calibration_loader</span><span class="p">,</span> <span class="n">quant_model</span><span class="p">,</span> <span class="n">args</span><span class="p">):</span>
    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
        <span class="c1"># Put the model in calibration mode to collect statistics</span>
        <span class="c1"># Quantization is automatically disabled during the calibration, and re-enabled at the end</span>
        <span class="k">with</span> <span class="n">calibration_mode</span><span class="p">(</span><span class="n">quant_model</span><span class="p">):</span>
            <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">images</span><span class="p">,</span> <span class="n">_</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">calibration_loader</span><span class="p">):</span>
                <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Calibration iteration </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
                <span class="n">quant_model</span><span class="p">(</span><span class="n">images</span><span class="p">)</span>

        <span class="c1"># Apply bias correction</span>
        <span class="k">with</span> <span class="n">bias_correction_mode</span><span class="p">(</span><span class="n">quant_model</span><span class="p">):</span>
            <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="p">(</span><span class="n">images</span><span class="p">,</span> <span class="n">_</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">calibration_loader</span><span class="p">):</span>
                <span class="nb">print</span><span class="p">(</span><span class="sa">f</span><span class="s1">&#39;Bias correction iteration </span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">)</span>
                <span class="n">quant_model</span><span class="p">(</span><span class="n">images</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">quant_model</span>
</pre></div>
</div>
</div>
</section>


                </article>
              
              
              
              
              
                <footer class="prev-next-footer d-print-none">
                  
<div class="prev-next-area">
    <a class="left-prev"
       href="index.html"
       title="previous page">
      <i class="fa-solid fa-angle-left"></i>
      <div class="prev-next-info">
        <p class="prev-next-subtitle">previous</p>
        <p class="prev-next-title">Tutorials</p>
      </div>
    </a>
    <a class="right-next"
       href="quant_tensor_quant_conv2d_overview.html"
       title="next page">
      <div class="prev-next-info">
        <p class="prev-next-subtitle">next</p>
        <p class="prev-next-title">An overview of QuantTensor and QuantConv2d</p>
      </div>
      <i class="fa-solid fa-angle-right"></i>
    </a>
</div>
                </footer>
              
            </div>
            
            
              
                <div class="bd-sidebar-secondary bd-toc"><div class="sidebar-secondary-items sidebar-secondary__inner">


  <div class="sidebar-secondary-item">
<div
    id="pst-page-navigation-heading-2"
    class="page-toc tocsection onthispage">
    <i class="fa-solid fa-list"></i> On this page
  </div>
  <nav class="bd-toc-nav page-toc" aria-labelledby="pst-page-navigation-heading-2">
    <ul class="visible nav section-nav flex-column">
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#">Brevitas TVMCon 2021 tutorial</a></li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Fundamentals">Fundamentals</a><ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#QuantLinear-layer">QuantLinear layer</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Weight-quantization">Weight quantization</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Default-weight-quantization">Default weight quantization</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Mixing-quantized-weights-and-floating-point-inputs">Mixing quantized weights and floating-point inputs</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Fixed-point-weight-quantization">Fixed-point weight quantization</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Binary-weight-quantization">Binary weight quantization</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Sharing-a-weight-quantizer">Sharing a weight quantizer</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Inputs/Outputs/Activations-quantization:">Inputs/Outputs/Activations quantization:</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#QuantIdentity-layer">QuantIdentity layer</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#QuantReLU-layer">QuantReLU layer</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Requantizing-a-tensor">Requantizing a tensor</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#How-is-the-activation-scale-determined-by-default?">How is the activation scale determined by default?</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Bias-Quantization">Bias Quantization</a></li>
</ul>
</li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Operations-on-QuantTensor">Operations on QuantTensor</a><ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Element-wise-Arithmetic">Element-wise Arithmetic</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Element-wise-adds">Element-wise adds</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Calling-torch-functions">Calling torch functions</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#max_pool-on-QuantTensor">max_pool on QuantTensor</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#tanh-on-QuantTensor">tanh on QuantTensor</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#QuantTensor-concatenation">QuantTensor concatenation</a></li>
</ul>
</li>
</ul>
</li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Customizing-Quantizers">Customizing Quantizers</a><ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Common-keyword-arguments">Common keyword arguments</a><ul class="nav section-nav flex-column">
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Weight-bit-width">Weight bit-width</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Per-channel-weight-quantization">Per-channel weight quantization</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Activation-bit-width">Activation bit-width</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Activation-quantization-with-max_val-init">Activation quantization with max_val init</a></li>
<li class="toc-h3 nav-item toc-entry"><a class="reference internal nav-link" href="#Per-channel-activation-quantization">Per-channel activation quantization</a></li>
</ul>
</li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Inheriting-from-a-quantizer">Inheriting from a quantizer</a></li>
</ul>
</li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Defining-a-quantizer-from-scratch-with-enums">Defining a quantizer from scratch with enums</a><ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Weight-quantizer">Weight quantizer</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Activation-quantizer">Activation quantizer</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Learned-scale-and-bit-width-quantizer">Learned scale and bit-width quantizer</a></li>
</ul>
</li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Retraining-from-floating-point">Retraining from floating-point</a></li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Defining-a-quantizer-from-scratch-with-dependency-injection">Defining a quantizer from scratch with dependency-injection</a><ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Activation-quantization-from-scratch">Activation quantization from scratch</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Weight-quantization-with-learned-scale-from-scratch">Weight quantization with learned scale from scratch</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Sharing-learned-bit-width-among-layers">Sharing learned bit-width among layers</a></li>
</ul>
</li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Export">Export</a><ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Export-to-ONNX-QOps">Export to ONNX QOps</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Export-to-custom-Quantized-ONNX">Export to custom Quantized ONNX</a></li>
<li class="toc-h2 nav-item toc-entry"><a class="reference internal nav-link" href="#Export-to-TorchScript-quantization-backend">Export to TorchScript quantization backend</a></li>
</ul>
</li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Brevitas-and-FX">Brevitas and FX</a></li>
<li class="toc-h1 nav-item toc-entry"><a class="reference internal nav-link" href="#Calibration-based-post-training-quantization">Calibration-based post-training quantization</a></li>
</ul>

  </nav></div>

  <div class="sidebar-secondary-item">

  <div class="tocsection sourcelink">
    <a href="../_sources/tutorials/tvmcon2021.nblink.txt">
      <i class="fa-solid fa-file-lines"></i> Show Source
    </a>
  </div>
</div>

</div></div>
              
            
          </div>
          <footer class="bd-footer-content">
            
          </footer>
        
      </main>
    </div>
  </div>
  
  <!-- Scripts loaded after <body> so the DOM is not blocked -->
  <script src="../_static/scripts/bootstrap.js?digest=3ee479438cf8b5e0d341"></script>
<script src="../_static/scripts/pydata-sphinx-theme.js?digest=3ee479438cf8b5e0d341"></script>

  <footer class="bd-footer">
<div class="bd-footer__inner bd-page-width">
  
    <div class="footer-items__start">
      
        <div class="footer-item">

  <p class="copyright">
    
      © Copyright 2025 - Advanced Micro Devices, Inc..
      <br/>
    
  </p>
</div>
      
        <div class="footer-item">

  <p class="sphinx-version">
    Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 5.3.0.
    <br/>
  </p>
</div>
      
    </div>
  
  
  
    <div class="footer-items__end">
      
        <div class="footer-item">
<script>
document.write(`
  <div class="version-switcher__container dropdown">
    <button id="pst-version-switcher-button-2"
      type="button"
      class="version-switcher__button btn btn-sm navbar-btn dropdown-toggle"
      data-bs-toggle="dropdown"
      aria-haspopup="listbox"
      aria-controls="pst-version-switcher-list-2"
      aria-label="Version switcher list"
    >
      Choose version  <!-- this text may get changed later by javascript -->
      <span class="caret"></span>
    </button>
    <div id="pst-version-switcher-list-2"
      class="version-switcher__menu dropdown-menu list-group-flush py-0"
      role="listbox" aria-labelledby="pst-version-switcher-button-2">
      <!-- dropdown will be populated by javascript on page load -->
    </div>
  </div>
`);
</script></div>
      
    </div>
  
</div>

  </footer>
  </body>
</html>